78#define DEBUG_TYPE "dagcombine"
80STATISTIC(NodesCombined ,
"Number of dag nodes combined");
81STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
82STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
83STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
84STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
86STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
90 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
94 cl::desc(
"Enable DAG combiner's use of TBAA"));
99 cl::desc(
"Only use DAG-combiner alias analysis in this"
107 cl::desc(
"Bypass the profitability model of load slicing"),
112 cl::desc(
"DAG combiner may split indexing from loads"));
116 cl::desc(
"Limit the number of operands to inline for Token Factors"));
125 bool LegalOperations =
false;
126 bool LegalTypes =
false;
160 void AddUsersToWorklist(
SDNode *
N) {
168 void clearAddedDanglingWorklistEntries() {
170 while (!PruningList.
empty()) {
173 recursivelyDeleteUnusedNodes(
N);
177 SDNode *getNextWorklistEntry() {
179 clearAddedDanglingWorklistEntries();
183 while (!
N && !Worklist.
empty()) {
188 bool GoodWorklistEntry = WorklistMap.
erase(
N);
189 (void)GoodWorklistEntry;
190 assert(GoodWorklistEntry &&
191 "Found a worklist entry without a corresponding map entry!");
202 OptLevel(OL), AA(AA) {
205 MaximumLegalStoreInBits = 0;
209 VT.getSizeInBits() >= MaximumLegalStoreInBits)
210 MaximumLegalStoreInBits = VT.getSizeInBits();
213 void ConsiderForPruning(
SDNode *
N) {
220 void AddToWorklist(
SDNode *
N) {
222 "Deleted Node added to Worklist");
229 ConsiderForPruning(
N);
231 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
236 void removeFromWorklist(
SDNode *
N) {
240 auto It = WorklistMap.
find(
N);
241 if (It == WorklistMap.
end())
245 Worklist[It->second] =
nullptr;
246 WorklistMap.
erase(It);
249 void deleteAndRecombine(
SDNode *
N);
250 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
258 return CombineTo(
N, &Res, 1, AddTo);
265 return CombineTo(
N, To, 2, AddTo);
271 unsigned MaximumLegalStoreInBits;
276 bool SimplifyDemandedBits(
SDValue Op) {
277 unsigned BitWidth =
Op.getScalarValueSizeInBits();
283 EVT VT =
Op.getValueType();
286 return SimplifyDemandedBits(Op,
DemandedBits, DemandedElts);
292 bool SimplifyDemandedVectorElts(
SDValue Op) {
293 unsigned NumElts =
Op.getValueType().getVectorNumElements();
295 return SimplifyDemandedVectorElts(Op, DemandedElts);
299 const APInt &DemandedElts);
300 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
301 bool AssumeSingleUse =
false);
303 bool CombineToPreIndexedLoadStore(
SDNode *
N);
304 bool CombineToPostIndexedLoadStore(
SDNode *
N);
324 void ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad);
465 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
483 bool NotExtCompare =
false);
484 SDValue convertSelectOfFPConstantsToLoadOffset(
494 const SDLoc &DL,
bool foldBooleans);
499 bool isOneUseSetCC(
SDValue N)
const;
522 bool DemandHighBits =
true);
526 unsigned PosOpcode,
unsigned NegOpcode,
540 SDValue VecIn2,
unsigned LeftIdx,
575 int64_t OffsetFromBase;
578 : MemNode(
N), OffsetFromBase(
Offset) {}
585 bool isMulAddWithConstProfitable(
SDNode *MulNode,
593 EVT LoadResultTy,
EVT &ExtVT);
598 EVT &MemVT,
unsigned ShAmt = 0);
619 EVT MemVT,
unsigned NumStores,
620 bool IsConstantSrc,
bool UseVector,
636 bool checkMergeStoreCandidatesForDependencies(
658 bool hasOperation(
unsigned Opcode,
EVT VT) {
672 EVT getShiftAmountTy(
EVT LHSTy) {
679 bool isTypeLegal(
const EVT &VT) {
680 if (!LegalTypes)
return true;
685 EVT getSetCCResultType(
EVT VT)
const {
700 explicit WorklistRemover(DAGCombiner &dc)
701 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
704 DC.removeFromWorklist(
N);
712 explicit WorklistInserter(DAGCombiner &dc)
713 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
727 ((DAGCombiner*)
DC)->AddToWorklist(
N);
732 return ((DAGCombiner*)
DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
737 return ((DAGCombiner*)
DC)->CombineTo(
N, Res, AddTo);
742 return ((DAGCombiner*)
DC)->CombineTo(
N, Res0, Res1, AddTo);
747 return ((DAGCombiner*)
DC)->CommitTargetLoweringOpt(TLO);
754void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
755 removeFromWorklist(
N);
763 if (Op->hasOneUse() || Op->getNumValues() > 1)
764 AddToWorklist(Op.getNode());
776 unsigned Depth = 0) {
782 EVT VT = Op.getValueType();
784 if (!Op.hasOneUse() &&
786 TLI.
isFPExtFree(VT, Op.getOperand(0).getValueType())))
793 switch (Op.getOpcode()) {
794 default:
return false;
796 if (!LegalOperations)
808 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
811 if (!LegalOperations)
817 return N.isUndef() ||
818 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
832 Options, ForCodeSize,
Depth + 1))
836 ForCodeSize,
Depth + 1);
849 Options, ForCodeSize,
Depth + 1))
853 ForCodeSize,
Depth + 1);
859 ForCodeSize,
Depth + 1);
865 bool LegalOperations,
bool ForCodeSize,
866 unsigned Depth = 0) {
869 return Op.getOperand(0);
871 assert(
Depth <= 6 &&
"GetNegatedExpression doesn't match isNegatibleForFree");
875 switch (Op.getOpcode()) {
878 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
889 APFloat V = cast<ConstantFPSDNode>(
C)->getValueAPF();
904 LegalOperations, ForCodeSize,
906 Op.getOperand(1), Flags);
910 LegalOperations, ForCodeSize,
912 Op.getOperand(0), Flags);
918 return Op.getOperand(1);
922 Op.getOperand(1), Op.getOperand(0), Flags);
930 return DAG.
getNode(Op.getOpcode(),
SDLoc(Op), Op.getValueType(),
932 LegalOperations, ForCodeSize,
934 Op.getOperand(1), Flags);
937 return DAG.
getNode(Op.getOpcode(),
SDLoc(Op), Op.getValueType(),
940 LegalOperations, ForCodeSize,
945 return DAG.
getNode(Op.getOpcode(),
SDLoc(Op), Op.getValueType(),
947 LegalOperations, ForCodeSize,
952 LegalOperations, ForCodeSize,
975 LHS =
N.getOperand(0);
976 RHS =
N.getOperand(1);
977 CC =
N.getOperand(2);
990 LHS =
N.getOperand(0);
991 RHS =
N.getOperand(1);
992 CC =
N.getOperand(4);
999bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
1001 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N.getNode()->hasOneUse())
1009 if (isa<ConstantFPSDNode>(
N))
1021 return !(Const->isOpaque() && NoOpaques);
1024 unsigned BitWidth =
N.getScalarValueSizeInBits();
1025 for (
const SDValue &Op :
N->op_values()) {
1029 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
1030 (Const->isOpaque() && NoOpaques))
1045bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1061 auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
1062 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1066 const APInt &C1APIntVal = C1->getAPIntValue();
1067 const APInt &C2APIntVal = C2->getAPIntValue();
1071 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1074 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1084 AM.
BaseOffs = C2APIntVal.getSExtValue();
1086 unsigned AS =
LoadStore->getAddressSpace();
1103SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &DL,
1127 AddToWorklist(OpNode.
getNode());
1148 if (
SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1))
1150 if (
SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0))
1157 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1161 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1162 for (
unsigned i = 0, e = NumTo; i != e; ++i)
1163 assert((!To[i].getNode() ||
1164 N->getValueType(i) == To[i].getValueType()) &&
1165 "Cannot combine value to value of different type!");
1167 WorklistRemover DeadNodes(*
this);
1171 for (
unsigned i = 0, e = NumTo; i != e; ++i) {
1172 if (To[i].getNode()) {
1173 AddToWorklist(To[i].getNode());
1174 AddUsersToWorklist(To[i].getNode());
1183 deleteAndRecombine(
N);
1191 WorklistRemover DeadNodes(*
this);
1208 const APInt &DemandedElts) {
1215 AddToWorklist(
Op.getNode());
1223 CommitTargetLoweringOpt(TLO);
1231 const APInt &DemandedElts,
1232 bool AssumeSingleUse) {
1234 APInt KnownUndef, KnownZero;
1236 TLO, 0, AssumeSingleUse))
1240 AddToWorklist(
Op.getNode());
1248 CommitTargetLoweringOpt(TLO);
1252void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad) {
1254 EVT VT =
Load->getValueType(0);
1259 WorklistRemover DeadNodes(*
this);
1262 deleteAndRecombine(Load);
1263 AddToWorklist(Trunc.
getNode());
1271 EVT MemVT =
LD->getMemoryVT();
1273 :
LD->getExtensionType();
1276 LD->getChain(),
LD->getBasePtr(),
1277 MemVT,
LD->getMemOperand());
1280 unsigned Opc =
Op.getOpcode();
1284 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1288 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1294 return DAG.
getNode(ExtOpc, DL, PVT, Op);
1306 EVT OldVT =
Op.getValueType();
1308 bool Replace =
false;
1309 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1312 AddToWorklist(NewOp.
getNode());
1315 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1321 EVT OldVT =
Op.getValueType();
1323 bool Replace =
false;
1324 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1327 AddToWorklist(NewOp.
getNode());
1330 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1338 if (!LegalOperations)
1341 EVT VT =
Op.getValueType();
1347 unsigned Opc =
Op.getOpcode();
1355 assert(PVT != VT &&
"Don't know what type to promote to!");
1359 bool Replace0 =
false;
1361 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1363 bool Replace1 =
false;
1365 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1374 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1377 CombineTo(
Op.getNode(), RV);
1403 if (!LegalOperations)
1406 EVT VT =
Op.getValueType();
1412 unsigned Opc =
Op.getOpcode();
1420 assert(PVT != VT &&
"Don't know what type to promote to!");
1424 bool Replace =
false;
1428 N0 = SExtPromoteOperand(N0, PVT);
1430 N0 = ZExtPromoteOperand(N0, PVT);
1432 N0 = PromoteOperand(N0, PVT, Replace);
1443 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1453 if (!LegalOperations)
1456 EVT VT =
Op.getValueType();
1462 unsigned Opc =
Op.getOpcode();
1470 assert(PVT != VT &&
"Don't know what type to promote to!");
1480bool DAGCombiner::PromoteLoad(
SDValue Op) {
1481 if (!LegalOperations)
1487 EVT VT =
Op.getValueType();
1493 unsigned Opc =
Op.getOpcode();
1501 assert(PVT != VT &&
"Don't know what type to promote to!");
1506 EVT MemVT =
LD->getMemoryVT();
1508 :
LD->getExtensionType();
1510 LD->getChain(),
LD->getBasePtr(),
1511 MemVT,
LD->getMemOperand());
1515 Result.getNode()->dump(&DAG);
dbgs() <<
'\n');
1516 WorklistRemover DeadNodes(*
this);
1519 deleteAndRecombine(
N);
1520 AddToWorklist(
Result.getNode());
1532bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1533 if (!
N->use_empty())
1543 if (
N->use_empty()) {
1544 for (
const SDValue &ChildN :
N->op_values())
1545 Nodes.
insert(ChildN.getNode());
1547 removeFromWorklist(
N);
1552 }
while (!Nodes.
empty());
1566 WorklistInserter AddNodes(*
this);
1570 AddToWorklist(&
Node);
1578 while (
SDNode *
N = getNextWorklistEntry()) {
1582 if (recursivelyDeleteUnusedNodes(
N))
1585 WorklistRemover DeadNodes(*
this);
1593 for (
SDNode *LN : UpdatedNodes) {
1595 AddUsersToWorklist(LN);
1607 for (
const SDValue &ChildN :
N->op_values())
1608 if (!CombinedNodes.
count(ChildN.getNode()))
1609 AddToWorklist(ChildN.getNode());
1627 "Node was deleted but visit returned new node!");
1635 N->getNumValues() == 1 &&
"Type mismatch");
1641 AddUsersToWorklist(RV.
getNode());
1647 recursivelyDeleteUnusedNodes(
N);
1656 switch (
N->getOpcode()) {
1792 "Node was deleted but visit returned NULL!");
1799 DagCombineInfo(DAG, Level,
false,
this);
1807 switch (
N->getOpcode()) {
1815 RV = PromoteIntBinOp(
SDValue(
N, 0));
1820 RV = PromoteIntShiftOp(
SDValue(
N, 0));
1837 N->getNumValues() == 1) {
1842 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1857 if (
unsigned NumOps =
N->getNumOperands()) {
1858 if (
N->getOperand(0).getValueType() ==
MVT::Other)
1859 return N->getOperand(0);
1860 if (
N->getOperand(NumOps-1).getValueType() ==
MVT::Other)
1861 return N->getOperand(NumOps-1);
1862 for (
unsigned i = 1; i < NumOps-1; ++i)
1863 if (
N->getOperand(i).getValueType() ==
MVT::Other)
1864 return N->getOperand(i);
1872 if (
N->getNumOperands() == 2) {
1874 return N->getOperand(0);
1876 return N->getOperand(1);
1887 AddToWorklist(*(
N->use_begin()));
1892 bool Changed =
false;
1899 for (
unsigned i = 0; i < TFs.
size(); ++i) {
1904 for (
unsigned j = i; j < TFs.
size(); j++)
1915 switch (
Op.getOpcode()) {
1933 if (SeenOps.
insert(
Op.getNode()).second)
1944 for (
unsigned i = 1, e = TFs.
size(); i < e; i++)
1945 AddToWorklist(TFs[i]);
1957 bool DidPruneOps =
false;
1959 unsigned NumLeftToConsider = 0;
1960 for (
const SDValue &Op : Ops) {
1961 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
1965 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
1968 if (SeenOps.
count(Op) != 0) {
1971 unsigned OrigOpNumber = 0;
1972 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
1974 assert((OrigOpNumber != Ops.size()) &&
1975 "expected to find TokenFactor Operand");
1977 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
1978 if (Worklist[i].
second == OrigOpNumber) {
1979 Worklist[i].second = OpNumber;
1982 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1983 OpWorkCount[OrigOpNumber] = 0;
1984 NumLeftToConsider--;
1987 if (SeenChains.
insert(Op).second) {
1988 OpWorkCount[OpNumber]++;
1989 Worklist.
push_back(std::make_pair(Op, OpNumber));
1993 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
1995 if (NumLeftToConsider <= 1)
1997 auto CurNode = Worklist[i].first;
1998 auto CurOpNumber = Worklist[i].second;
1999 assert((OpWorkCount[CurOpNumber] > 0) &&
2000 "Node should not appear in worklist");
2001 switch (CurNode->getOpcode()) {
2007 NumLeftToConsider++;
2011 AddToWorklist(i,
Op.getNode(), CurOpNumber);
2017 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2020 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2021 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2024 OpWorkCount[CurOpNumber]--;
2025 if (OpWorkCount[CurOpNumber] == 0)
2026 NumLeftToConsider--;
2039 for (
const SDValue &Op : Ops) {
2040 if (SeenChains.
count(
Op.getNode()) == 0)
2055 WorklistRemover DeadNodes(*
this);
2061 AddUsersToWorklist(
N);
2065 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i)
2068 }
while (!
N->use_empty());
2069 deleteAndRecombine(
N);
2077 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2082 "Unexpected binary operator");
2087 unsigned SelOpNo = 0;
2113 bool CanFoldNonConst =
2119 if (!CanFoldNonConst &&
2139 SDValue NewCT = SelOpNo ? DAG.
getNode(BinOpcode, DL, VT, CBO, CT)
2140 : DAG.
getNode(BinOpcode, DL, VT, CT, CBO);
2141 if (!CanFoldNonConst && !NewCT.
isUndef() &&
2146 SDValue NewCF = SelOpNo ? DAG.
getNode(BinOpcode, DL, VT, CBO, CF)
2147 : DAG.
getNode(BinOpcode, DL, VT, CF, CBO);
2148 if (!CanFoldNonConst && !NewCF.
isUndef() &&
2160 "Expecting add or sub");
2165 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2166 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2167 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2168 auto *CN = dyn_cast<ConstantSDNode>(
C);
2174 Z.getOperand(0).getValueType() !=
MVT::i1)
2189 EVT VT =
C.getValueType();
2201 "Expecting add or sub");
2206 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2207 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2208 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2231 APInt NewC = IsAdd ?
C->getAPIntValue() + 1 :
C->getAPIntValue() - 1;
2246 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
2282 assert(Sub &&
"Constant folding failed");
2291 assert(Add &&
"Constant folding failed");
2302 if ((!LegalOperations ||
2305 X.getScalarValueSizeInBits() == 1) {
2321 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2325 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD, DL, N0, N1)) {
2392 return (!Max && !Op) ||
2393 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2447 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2450 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2462 if (
SDValue Combined = visitADDLike(
N))
2480 unsigned Opcode =
N->getOpcode();
2504 return DAG.
getNode(Opcode, DL, VT, N1, N0);
2523 bool Masked =
false;
2585 SDLoc DL(LocReference);
2655 DAG.
getVTList(VT, Carry.getValueType()), N0,
2668 if (!
N->hasAnyUseOfValue(1))
2719 if (Force && isa<ConstantSDNode>(V))
2731 bool IsFlip =
false;
2734 IsFlip = Const->isOne();
2737 IsFlip = Const->isAllOnesValue();
2740 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2757 EVT CarryVT =
N->getValueType(1);
2761 if (!
N->hasAnyUseOfValue(1))
2768 return DAG.
getNode(
N->getOpcode(), DL,
N->getVTList(), N1, N0);
2772 return CombineTo(
N, N0, DAG.
getConstant(0, DL, CarryVT));
2784 return CombineTo(
N, Sub,
2788 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
2791 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
2825 SDValue CarryIn =
N->getOperand(2);
2844 SDValue CarryIn =
N->getOperand(2);
2855 if (!LegalOperations ||
2865 AddToWorklist(CarryExt.
getNode());
2871 if (
SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn,
N))
2874 if (
SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn,
N))
2975 return CombineTo(
N, Sub,
3025 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
3044 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3063 if (ShiftAmt && ShiftAmt->
getAPIntValue() == (BitWidth - 1)) {
3071 if (
N->getFlags().hasNoUnsignedWrap())
3077 if (
N->getFlags().hasNoSignedWrap())
3111 assert(NewC &&
"Constant folding failed");
3122 assert(NewC &&
"Constant folding failed");
3133 assert(NewC &&
"Constant folding failed");
3143 assert(NewC &&
"Constant folding failed");
3265 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
3268 if (
C->getAPIntValue() == (OpSizeInBits - 1))
3284 if (GA->getGlobal() == GB->getGlobal())
3285 return DAG.
getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3358 if (!
N->hasAnyUseOfValue(1))
3385 EVT CarryVT =
N->getValueType(1);
3389 if (!
N->hasAnyUseOfValue(1))
3408 return CombineTo(
N, N0, DAG.
getConstant(0, DL, CarryVT));
3421 SDValue CarryIn =
N->getOperand(2);
3433 SDValue CarryIn =
N->getOperand(2);
3437 if (!LegalOperations ||
3454 bool N0IsConst =
false;
3455 bool N1IsConst =
false;
3456 bool N1IsOpaqueConst =
false;
3457 bool N0IsOpaqueConst =
false;
3458 APInt ConstValue0, ConstValue1;
3461 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
3468 "Splat APInt should be element width");
3471 "Splat APInt should be element width");
3473 N0IsConst = isa<ConstantSDNode>(N0);
3475 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
3476 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
3478 N1IsConst = isa<ConstantSDNode>(N1);
3480 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3481 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3486 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
3501 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3515 SDValue LogBase2 = BuildLogBase2(N1, DL);
3521 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
3522 unsigned Log2Val = (-ConstValue1).logBase2();
3546 if ((MulC - 1).isPowerOf2())
3548 else if ((MulC + 1).isPowerOf2())
3553 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
3555 "multiply-by-constant generated out of bounds shift");
3578 SDValue Sh(
nullptr, 0),
Y(
nullptr, 0);
3601 isMulAddWithConstProfitable(
N, N0, N1))
3619 EVT NodeType =
Node->getValueType(0);
3620 if (!NodeType.isSimple())
3622 switch (NodeType.getSimpleVT().SimpleTy) {
3623 default:
return false;
3624 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
3625 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
3626 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
3627 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
3628 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
3636 if (
Node->use_empty())
3639 unsigned Opcode =
Node->getOpcode();
3644 EVT VT =
Node->getValueType(0);
3658 unsigned OtherOpcode = 0;
3681 unsigned UserOpc =
User->getOpcode();
3682 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
3686 if (UserOpc == OtherOpcode) {
3689 }
else if (UserOpc == DivRemOpc) {
3692 assert(UserOpc == Opcode);
3697 CombineTo(
User, combined);
3708 EVT VT =
N->getValueType(0);
3711 unsigned Opc =
N->getOpcode();
3720 if (DAG.
isUndef(Opc, {N0, N1}))
3754 EVT VT =
N->getValueType(0);
3759 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
3781 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3789 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
3798 CombineTo(RemNode, Sub);
3816 EVT VT =
N->getValueType(0);
3823 if (
C->isNullValue() ||
C->isOpaque())
3825 if (
C->getAPIntValue().isPowerOf2())
3827 if ((-
C->getAPIntValue()).isPowerOf2())
3853 AddToWorklist(Sign.
getNode());
3859 AddToWorklist(
Add.getNode());
3870 Sra = DAG.
getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
3898 EVT VT =
N->getValueType(0);
3903 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
3924 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3927 if (
SDValue V = visitUDIVLike(N0, N1,
N)) {
3936 CombineTo(RemNode, Sub);
3954 EVT VT =
N->getValueType(0);
3959 SDValue LogBase2 = BuildLogBase2(N1, DL);
3960 AddToWorklist(LogBase2.
getNode());
3964 AddToWorklist(Trunc.
getNode());
3973 SDValue LogBase2 = BuildLogBase2(N10, DL);
3974 AddToWorklist(LogBase2.
getNode());
3978 AddToWorklist(Trunc.
getNode());
3980 AddToWorklist(
Add.getNode());
3997 unsigned Opcode =
N->getOpcode();
4000 EVT VT =
N->getValueType(0);
4020 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4033 AddToWorklist(
Add.getNode());
4040 AddToWorklist(
Add.getNode());
4056 isSigned ? visitSDIVLike(N0, N1,
N) : visitUDIVLike(N0, N1,
N);
4062 CombineTo(DivNode, OptimizedDiv);
4065 AddToWorklist(OptimizedDiv.
getNode());
4073 return DivRem.getValue(1);
4081 EVT VT =
N->getValueType(0);
4128 EVT VT =
N->getValueType(0);
4153 SDValue LogBase2 = BuildLogBase2(N1, DL);
4184SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *
N,
unsigned LoOp,
4187 bool HiExists =
N->hasAnyUseOfValue(1);
4188 if (!HiExists && (!LegalOperations ||
4191 return CombineTo(
N, Res, Res);
4195 bool LoExists =
N->hasAnyUseOfValue(0);
4196 if (!LoExists && (!LegalOperations ||
4199 return CombineTo(
N, Res, Res);
4203 if (LoExists && HiExists)
4209 AddToWorklist(
Lo.getNode());
4212 (!LegalOperations ||
4214 return CombineTo(
N, LoOpt, LoOpt);
4219 AddToWorklist(
Hi.getNode());
4221 if (HiOpt.
getNode() && HiOpt != Hi &&
4222 (!LegalOperations ||
4224 return CombineTo(
N, HiOpt, HiOpt);
4234 EVT VT =
N->getValueType(0);
4254 return CombineTo(
N, Lo, Hi);
4265 EVT VT =
N->getValueType(0);
4285 return CombineTo(
N, Lo, Hi);
4297 if (C2->getAPIntValue() == 2)
4299 N->getVTList(),
N->getOperand(0),
N->getOperand(0));
4311 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
4327 unsigned Opcode =
N->getOpcode();
4350 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
4352 unsigned LogicOpcode =
N->getOpcode();
4355 LogicOpcode ==
ISD::XOR) &&
"Expected logic opcode");
4368 EVT XVT =
X.getValueType();
4377 if (XVT !=
Y.getValueType())
4381 if ((VT.
isVector() || LegalOperations) &&
4391 return DAG.
getNode(HandOpcode, DL, VT, Logic);
4401 if (XVT !=
Y.getValueType())
4413 return DAG.
getNode(HandOpcode, DL, VT, Logic);
4434 return DAG.
getNode(HandOpcode, DL, VT, Logic);
4447 if (XVT.
isInteger() && XVT ==
Y.getValueType()) {
4449 return DAG.
getNode(HandOpcode, DL, VT, Logic);
4466 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
4467 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
4468 assert(
X.getValueType() ==
Y.getValueType() &&
4469 "Inputs to shuffles are not the same type");
4475 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
4476 !SVN0->getMask().equals(SVN1->getMask()))
4512 SDValue LL, LR, RL, RR, N0CC, N1CC;
4513 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
4514 !isSetCCEquivalent(N1, RL, RR, N1CC))
4518 "Unexpected operand types for bitwise logic op");
4521 "Unexpected operand types for setcc");
4537 if (LR == RR && CC0 == CC1 && IsInteger) {
4542 bool AndEqZero = IsAnd && CC1 ==
ISD::SETEQ && IsZero;
4544 bool AndGtNeg1 = IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
4546 bool OrNeZero = !IsAnd && CC1 ==
ISD::SETNE && IsZero;
4548 bool OrLtZero = !IsAnd && CC1 ==
ISD::SETLT && IsZero;
4554 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
4557 return DAG.
getSetCC(DL, VT, Or, LR, CC1);
4561 bool AndEqNeg1 = IsAnd && CC1 ==
ISD::SETEQ && IsNeg1;
4563 bool AndLtZero = IsAnd && CC1 ==
ISD::SETLT && IsZero;
4565 bool OrNeNeg1 = !IsAnd && CC1 ==
ISD::SETNE && IsNeg1;
4567 bool OrGtNeg1 = !IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
4573 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
4575 AddToWorklist(And.getNode());
4576 return DAG.
getSetCC(DL, VT, And, LR, CC1);
4589 AddToWorklist(
Add.getNode());
4604 return DAG.
getSetCC(DL, VT, Or, Zero, CC1);
4621 if ((C0Val - C1Val).isPowerOf2()) {
4629 return DAG.
getSetCC(DL, VT, And, Zero, CC0);
4636 if (LL == RR && LR == RL) {
4643 if (LL == RL && LR == RR) {
4647 (!LegalOperations ||
4650 return DAG.
getSetCC(DL, VT, LL, LR, NewCC);
4668 if (
SDValue V = foldLogicOfSetCCs(
true, N0, N1, DL))
4680 APInt SRLC = SRLI->getAPIntValue();
4681 if (
ADDC.getMinSignedBits() <= 64 &&
4693 CombineTo(N0.
getNode(), NewAdd);
4710 const APInt &AndMask = CAnd->getAPIntValue();
4723 (ShiftBits + MaskBits <=
Size / 2) &&
4756 EVT LoadResultTy,
EVT &ExtVT) {
4765 if (ExtVT == LoadedVT &&
4766 (!LegalOperations ||
4782 if (LegalOperations &&
4826 if (isa<LoadSDNode>(LDST)) {
4833 if (LegalOperations &&
4842 if (
Load->getNumValues() > 2)
4855 assert(isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode");
4861 if (LegalOperations &&
4868bool DAGCombiner::SearchForAndLoads(
SDNode *
N,
4875 for (
SDValue Op :
N->op_values()) {
4876 if (
Op.getValueType().isVector())
4880 if (
auto *
C = dyn_cast<ConstantSDNode>(Op)) {
4882 (
Mask->getAPIntValue() &
C->getAPIntValue()) !=
C->getAPIntValue())
4887 if (!
Op.hasOneUse())
4890 switch(
Op.getOpcode()) {
4892 auto *
Load = cast<LoadSDNode>(Op);
4894 if (isAndLoadExtLoad(Mask, Load,
Load->getValueType(0), ExtVT) &&
4912 unsigned ActiveBits =
Mask->getAPIntValue().countTrailingOnes();
4915 cast<VTSDNode>(
Op.getOperand(1))->getVT() :
4916 Op.getOperand(0).getValueType();
4927 if (!SearchForAndLoads(
Op.getNode(), Loads, NodesWithConsts, Mask,
4938 NodeToMask =
Op.getNode();
4941 for (
unsigned i = 0, e = NodeToMask->
getNumValues(); i < e; ++i) {
4945 NodeToMask =
nullptr;
4958 auto *
Mask = dyn_cast<ConstantSDNode>(
N->getOperand(1));
4962 if (!
Mask->getAPIntValue().isMask())
4966 if (isa<LoadSDNode>(
N->getOperand(0)))
4971 SDNode *FixupNode =
nullptr;
4972 if (SearchForAndLoads(
N, Loads, NodesWithConsts, Mask, FixupNode)) {
4973 if (Loads.
size() == 0)
4985 SDValue(FixupNode, 0), MaskOp);
4987 if (And.getOpcode() == ISD ::AND)
4992 for (
auto *LogicN : NodesWithConsts) {
4996 if (isa<ConstantSDNode>(Op0))
5006 for (
auto *Load : Loads) {
5011 if (And.getOpcode() == ISD ::AND)
5014 SDValue NewLoad = ReduceLoadWidth(And.getNode());
5016 "Shouldn't be masking the load if it can't be narrowed");
5017 CombineTo(Load, NewLoad, NewLoad.
getValue(1));
5030SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(
SDNode *
N) {
5041 unsigned OuterShift;
5042 unsigned InnerShift;
5044 auto matchMask = [&OuterShift, &InnerShift, &
Y](
SDValue M) ->
bool {
5047 OuterShift =
M->getOpcode();
5056 Y =
M->getOperand(1);
5063 else if (matchMask(N0))
5069 EVT VT =
N->getValueType(0);
5090 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
5113 if (N0C && N1C && !N1C->
isOpaque())
5128 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
5137 if (
SDValue Shuffle = XformToShuffleWithZero(
N))
5142 return RHS->getAPIntValue().isSubsetOf(LHS->
getAPIntValue());
5162 CombineTo(N0.
getNode(), Zext);
5187 APInt SplatValue, SplatUndef;
5188 unsigned SplatBitSize;
5190 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
5191 SplatBitSize, HasAnyUndefs);
5195 SplatValue |= SplatUndef;
5201 unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
5206 if (EltBitWidth > SplatBitSize)
5207 for (SplatValue = SplatValue.
zextOrTrunc(EltBitWidth);
5208 SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
5209 SplatValue |= SplatValue.
shl(SplatBitSize);
5213 if ((SplatBitSize % EltBitWidth) == 0) {
5215 for (
unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
5225 Load->getValueType(0),
5226 Load->getMemoryVT());
5234 switch (
Load->getExtensionType()) {
5235 default:
B =
false;
break;
5247 CombineTo(
N, (N0.
getNode() == Load) ? NewLoad : N0);
5252 Load->getChain(),
Load->getBasePtr(),
5253 Load->getOffset(),
Load->getMemoryVT(),
5254 Load->getMemOperand());
5256 if (
Load->getNumValues() == 3) {
5260 CombineTo(Load, To, 3,
true);
5262 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
5276 if (
SDValue Res = ReduceLoadWidth(
N)) {
5290 if (BackwardsPropagateMask(
N, DAG)) {
5295 if (
SDValue Combined = visitANDLike(N0, N1,
N))
5300 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
5358 if (
SDValue Shifts = unfoldExtremeBitClearingToShifts(
N))
5366 bool DemandHighBits) {
5367 if (!LegalOperations)
5370 EVT VT =
N->getValueType(0);
5377 bool LookPassAnd0 =
false;
5378 bool LookPassAnd1 =
false;
5393 LookPassAnd0 =
true;
5403 LookPassAnd1 =
true;
5429 LookPassAnd0 =
true;
5443 LookPassAnd1 =
true;
5452 if (DemandHighBits && OpSizeInBits > 16) {
5461 if (!LookPassAnd1 &&
5468 if (OpSizeInBits > 16) {
5484 if (!
N.getNode()->hasOneUse())
5487 unsigned Opc =
N.getOpcode();
5499 N1C = dyn_cast<ConstantSDNode>(
N.getOperand(1));
5501 N1C = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
5505 unsigned MaskByteOffset;
5509 case 0xFF: MaskByteOffset = 0;
break;
5510 case 0xFF00: MaskByteOffset = 1;
break;
5519 case 0xFF0000: MaskByteOffset = 2;
break;
5520 case 0xFF000000: MaskByteOffset = 3;
break;
5525 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
5531 if (!
C ||
C->getZExtValue() != 8)
5539 if (!
C ||
C->getZExtValue() != 8)
5545 if (MaskByteOffset != 0 && MaskByteOffset != 2)
5548 if (!
C ||
C->getZExtValue() != 8)
5553 if (MaskByteOffset != 1 && MaskByteOffset != 3)
5556 if (!
C ||
C->getZExtValue() != 8)
5560 if (Parts[MaskByteOffset])
5574 if (!LegalOperations)
5577 EVT VT =
N->getValueType(0);
5623 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
5652 if (
SDValue V = foldLogicOfSetCCs(
false, N0, N1, DL))
5667 const APInt &LHSMask = N0O1C->getAPIntValue();
5668 const APInt &RHSMask = N1O1C->getAPIntValue();
5723 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
5742 if (isa<ShuffleVectorSDNode>(N0) &&
5743 isa<ShuffleVectorSDNode>(N1) &&
5751 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
5752 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
5753 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
5756 bool CanFold =
true;
5760 for (
int i = 0; i != NumElts; ++i) {
5765 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
5766 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
5770 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
5776 if (M0Zero == M1Zero) {
5781 assert((M0 >= 0 || M1 >= 0) &&
"Undef index!");
5787 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
5811 if (N0C && N1C && !N1C->
isOpaque())
5824 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
5831 if (
SDValue Combined = visitORLike(N0, N1,
N))
5835 if (
SDValue BSwap = MatchBSwapHWord(
N, N0, N1))
5837 if (
SDValue BSwap = MatchBSwapHWordLow(
N, N0, N1))
5866 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
5873 if (
SDValue Load = MatchLoadCombine(
N))
5883 if (
SDValue Combined = visitADDLike(
N))
5892 Mask = Op.getOperand(1);
5893 return Op.getOperand(0);
5933 assert(OppShift && ExtractFrom &&
"Empty SDValue");
5936 "Existing shift must be valid as a rotate half");
5944 bool IsMulOrDiv =
false;
5947 auto SelectOpcode = [&](
unsigned NeededShift,
unsigned MulOrDivVariant) {
5948 IsMulOrDiv = ExtractFrom.
getOpcode() == MulOrDivVariant;
5949 if (!IsMulOrDiv && ExtractFrom.
getOpcode() != NeededShift)
5951 Opcode = NeededShift;
6005 if (Rem != 0 || ResultAmt != OppLHSAmt)
6011 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.
zextOrTrunc(
6020 return DAG.
getNode(Opcode, DL, ResVT, OppShiftLHS, NewShiftNode);
6065 unsigned MaskLoBits = 0;
6069 unsigned Bits =
Log2_64(EltSize);
6070 if (NegC->getAPIntValue().getActiveBits() <= Bits &&
6071 ((NegC->getAPIntValue() | Known.
Zero).countTrailingOnes() >= Bits)) {
6091 if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
6092 ((PosC->getAPIntValue() | Known.
Zero).countTrailingOnes() >=
6131 return Width.getLoBits(MaskLoBits) == 0;
6132 return Width == EltSize;
6142 SDValue InnerNeg,
unsigned PosOpcode,
6143 unsigned NegOpcode,
const SDLoc &DL) {
6154 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
6155 HasPos ? Pos : Neg).
getNode();
6170 bool HasROTL = hasOperation(
ISD::ROTL, VT);
6171 bool HasROTR = hasOperation(
ISD::ROTR, VT);
6172 if (!HasROTL && !HasROTR)
return nullptr;
6194 if (!LHSShift && !RHSShift)
6209 RHSShift = NewRHSShift;
6214 LHSShift = NewLHSShift;
6217 if (!RHSShift || !LHSShift)
6245 return (LHS->
getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
6249 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
6279 SDValue LExtOp0 = LHSShiftAmt;
6280 SDValue RExtOp0 = RHSShiftAmt;
6293 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
6298 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
6310struct ByteProvider {
6315 unsigned ByteOffset = 0;
6317 ByteProvider() =
default;
6320 return ByteProvider(Load, ByteOffset);
6323 static ByteProvider getConstantZero() {
return ByteProvider(
nullptr, 0); }
6325 bool isConstantZero()
const {
return !
Load; }
6326 bool isMemory()
const {
return Load; }
6333 ByteProvider(
LoadSDNode *Load,
unsigned ByteOffset)
6334 :
Load(
Load), ByteOffset(ByteOffset) {}
6352 bool Root =
false) {
6357 if (!Root && !Op.hasOneUse())
6360 assert(Op.getValueType().isScalarInteger() &&
"can't handle other types");
6361 unsigned BitWidth = Op.getValueSizeInBits();
6362 if (BitWidth % 8 != 0)
6364 unsigned ByteWidth = BitWidth / 8;
6365 assert(Index < ByteWidth &&
"invalid index requested");
6368 switch (Op.getOpcode()) {
6377 if (LHS->isConstantZero())
6379 if (RHS->isConstantZero())
6384 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
6388 uint64_t BitShift = ShiftOp->getZExtValue();
6389 if (BitShift % 8 != 0)
6391 uint64_t ByteShift = BitShift / 8;
6393 return Index < ByteShift
6394 ? ByteProvider::getConstantZero()
6401 SDValue NarrowOp = Op->getOperand(0);
6403 if (NarrowBitWidth % 8 != 0)
6405 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
6407 if (Index >= NarrowByteWidth)
6417 auto L = cast<LoadSDNode>(Op.getNode());
6418 if (L->isVolatile() || L->isIndexed())
6421 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
6422 if (NarrowBitWidth % 8 != 0)
6424 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
6426 if (Index >= NarrowByteWidth)
6449 int64_t FirstOffset) {
6451 unsigned Width = ByteOffsets.
size();
6455 bool BigEndian =
true, LittleEndian =
true;
6456 for (
unsigned i = 0; i < Width; i++) {
6457 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
6460 if (!BigEndian && !LittleEndian)
6464 assert((BigEndian != LittleEndian) &&
"It should be either big endian or"
6470 switch (
Value.getOpcode()) {
6511 Chain =
Store->getChain();
6516 *DAG.
getContext(), Width *
N->getMemoryVT().getSizeInBits());
6532 for (
auto Store : Stores) {
6545 dyn_cast<ConstantSDNode>(
Value.getOperand(1));
6561 CombinedValue =
Value;
6568 if (
Value.getValueType() == VT ||
6570 CombinedValue =
Value;
6578 int64_t ByteOffsetFromBase = 0;
6581 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
6585 if (ByteOffsetFromBase < FirstOffset) {
6587 FirstOffset = ByteOffsetFromBase;
6591 if (Offset < 0 || Offset >= Width || ByteOffsets[Offset] !=
INT64_MAX)
6593 ByteOffsets[
Offset] = ByteOffsetFromBase;
6597 assert(FirstStore &&
"First store must be set");
6622 if (!Allowed || !Fast)
6627 "Get unexpected store value to combine");
6676 "Can only match load combining against OR nodes");
6679 EVT VT =
N->getValueType(0);
6692 auto MemoryByteOffset = [&] (ByteProvider
P) {
6693 assert(
P.isMemory() &&
"Must be a memory byte provider");
6694 unsigned LoadBitWidth =
P.Load->getMemoryVT().getSizeInBits();
6695 assert(LoadBitWidth % 8 == 0 &&
6696 "can only analyze providers for individual bytes not bit");
6697 unsigned LoadByteWidth = LoadBitWidth / 8;
6698 return IsBigEndianTarget
6713 for (
unsigned i = 0; i < ByteWidth; i++) {
6715 if (!
P || !
P->isMemory())
6720 "Must be enforced by calculateByteProvider");
6727 else if (Chain != LChain)
6732 int64_t ByteOffsetFromBase = 0;
6735 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
6739 ByteOffsetFromBase += MemoryByteOffset(*
P);
6740 ByteOffsets[i] = ByteOffsetFromBase;
6743 if (ByteOffsetFromBase < FirstOffset) {
6744 FirstByteProvider =
P;
6745 FirstOffset = ByteOffsetFromBase;
6750 assert(!Loads.
empty() &&
"All the bytes of the value must be loaded from "
6751 "memory, so there must be at least one load which produces the value");
6752 assert(Base &&
"Base address of the accessed memory location must be set");
6761 assert(FirstByteProvider &&
"must be set");
6765 if (MemoryByteOffset(*FirstByteProvider) != 0)
6767 LoadSDNode *FirstLoad = FirstByteProvider->Load;
6773 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
6785 if (!Allowed || !Fast)
6819 EVT VT =
N->getValueType(0);
6825 if (And.getOpcode() !=
ISD::AND || !And.hasOneUse())
6827 SDValue Xor = And.getOperand(XorIdx);
6841 M = And.getOperand(XorIdx ? 0 : 1);
6847 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
6848 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
6854 if (isa<ConstantSDNode>(
M.getNode()))
6888 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
6920 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
6933 if (!LegalOperations ||
6949 isSetCCEquivalent(N0.
getOperand(0), LHS, RHS, CC)){
6962 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
6967 return DAG.
getNode(NewOpcode, DL, VT, LHS, RHS);
6974 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
6979 return DAG.
getNode(NewOpcode, DL, VT, LHS, RHS);
6996 AddToWorklist(NotX.
getNode());
7004 if (XorC && ShiftC) {
7008 if (ShiftAmt < BitWidth) {
7010 Ones = N0Opcode ==
ISD::SHL ? Ones.
shl(ShiftAmt) : Ones.
lshr(ShiftAmt);
7027 SDValue A0 =
A.getOperand(0), A1 =
A.getOperand(1);
7029 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0)) {
7032 if (
C->getAPIntValue() == (OpSizeInBits - 1))
7068 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7072 if (
SDValue MM = unfoldMaskedMerge(
N))
7094 SDNode *LHS =
N->getOperand(0).getNode();
7126 isa<ConstantSDNode>(BinOpLHSVal.
getOperand(1));
7130 if (!IsShiftByConstant && !IsCopyOrSelect)
7133 if (IsCopyOrSelect &&
N->hasOneUse())
7136 EVT VT =
N->getValueType(0);
7145 assert(isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!");
7161 EVT TruncVT =
N->getValueType(0);
7162 if (
N->hasOneUse() &&
N->getOperand(0).hasOneUse() &&
7164 SDValue N01 =
N->getOperand(0).getOperand(1);
7167 SDValue N00 =
N->getOperand(0).getOperand(0);
7170 AddToWorklist(Trunc00.
getNode());
7171 AddToWorklist(Trunc01.
getNode());
7183 EVT VT =
N->getValueType(0);
7200 if (Cst->getAPIntValue().uge(Bitsize)) {
7201 uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
7202 return DAG.
getNode(
N->getOpcode(), dl, VT, N0,
7211 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, NewOp1);
7221 bool SameSide = (
N->getOpcode() == NextOp);
7227 ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
7249 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
7277 if (N0C && N1C && !N1C->
isOpaque())
7280 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7304 APInt c2 = RHS->getAPIntValue();
7306 return (c1 + c2).uge(OpSizeInBits);
7314 APInt c2 = RHS->getAPIntValue();
7316 return (c1 + c2).ult(OpSizeInBits);
7339 auto MatchOutOfRange = [OpSizeInBits, InnerBitwidth](
ConstantSDNode *LHS,
7341 APInt c1 = LHS->getAPIntValue();
7342 APInt c2 = RHS->getAPIntValue();
7344 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
7345 (c1 + c2).uge(OpSizeInBits);
7352 auto MatchInRange = [OpSizeInBits, InnerBitwidth](
ConstantSDNode *LHS,
7354 APInt c1 = LHS->getAPIntValue();
7355 APInt c2 = RHS->getAPIntValue();
7357 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
7358 (c1 + c2).ult(OpSizeInBits);
7381 APInt c2 = RHS->getAPIntValue();
7392 AddToWorklist(NewSHL.
getNode());
7403 uint64_t C1 = N0C1->getZExtValue();
7423 if (N0C1->getAPIntValue().ult(OpSizeInBits)) {
7434 Mask.lshrInPlace(c1 - c2);
7466 AddToWorklist(Shl0.
getNode());
7467 AddToWorklist(Shl1.
getNode());
7481 if (
SDValue NewSHL = visitShiftByConstant(
N, N1C))
7504 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
7512 if (N0C && N1C && !N1C->
isOpaque())
7515 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7526 if ((!LegalOperations ||
7542 APInt c2 = RHS->getAPIntValue();
7544 APInt Sum = c1 + c2;
7555 ShiftValue = ShiftValues[0];
7583 if ((ShiftAmt > 0) &&
7595 N->getValueType(0), Trunc);
7620 if (LargeShift->getAPIntValue() == TruncBits) {
7641 if (
SDValue NewSRA = visitShiftByConstant(
N, N1C))
7658 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
7666 if (N0C && N1C && !N1C->
isOpaque())
7669 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7682 APInt c2 = RHS->getAPIntValue();
7684 return (c1 + c2).uge(OpSizeInBits);
7692 APInt c2 = RHS->getAPIntValue();
7694 return (c1 + c2).ult(OpSizeInBits);
7715 if (c1 + OpSizeInBits == InnerShiftSize) {
7717 if (c1 + c2 >= InnerShiftSize)
7735 AddToWorklist(
Mask.getNode());
7755 AddToWorklist(SmallShift.
getNode());
7782 APInt UnknownBits = ~Known.Zero;
7799 AddToWorklist(
Op.getNode());
7822 if (
SDValue NewSRL = visitShiftByConstant(
N, N1C))
7826 if (
SDValue NarrowLoad = ReduceLoadWidth(
N))
7846 if (
N->hasOneUse()) {
7862 EVT VT =
N->getValueType(0);
7874 return IsFSHL ? N0 : N1;
7876 auto IsUndefOrZero = [](
SDValue V) {
7885 if (Cst->getAPIntValue().uge(BitWidth)) {
7886 uint64_t RotAmt = Cst->getAPIntValue().urem(BitWidth);
7891 unsigned ShAmt = Cst->getZExtValue();
7893 return IsFSHL ? N0 : N1;
7899 if (IsUndefOrZero(N0))
7901 DAG.
getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt,
7903 if (IsUndefOrZero(N1))
7905 DAG.
getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt,
7926 if (N0 == N1 && hasOperation(RotOpc, VT))
7938 EVT VT =
N->getValueType(0);
7954 EVT VT =
N->getValueType(0);
7967 EVT VT =
N->getValueType(0);
7980 EVT VT =
N->getValueType(0);
7997 EVT VT =
N->getValueType(0);
8007 EVT VT =
N->getValueType(0);
8024 EVT VT =
N->getValueType(0);
8034 EVT VT =
N->getValueType(0);
8060 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
8076 return DAG.
getNode(IEEEOpcode, DL, VT, LHS, RHS);
8080 return DAG.
getNode(Opcode, DL, VT, LHS, RHS);
8091 return DAG.
getNode(IEEEOpcode, DL, VT, LHS, RHS);
8095 return DAG.
getNode(Opcode, DL, VT, LHS, RHS);
8107 EVT VT =
N->getValueType(0);
8108 EVT CondVT = Cond.getValueType();
8114 auto *C1 = dyn_cast<ConstantSDNode>(N1);
8115 auto *C2 = dyn_cast<ConstantSDNode>(N2);
8123 if (CondVT ==
MVT::i1 && !LegalOperations) {
8124 if (C1->isNullValue() && C2->isOne()) {
8131 if (C1->isNullValue() && C2->isAllOnesValue()) {
8138 if (C1->isOne() && C2->isNullValue()) {
8144 if (C1->isAllOnesValue() && C2->isNullValue()) {
8155 if (C1->getAPIntValue() - 1 == C2->getAPIntValue()) {
8161 if (C1->getAPIntValue() + 1 == C2->getAPIntValue()) {
8186 C1->isNullValue() && C2->isOne()) {
8201 EVT VT =
N->getValueType(0);
8214 if (
SDValue V = foldSelectOfConstants(
N))
8220 AddToWorklist(NOTNode.
getNode());
8226 AddToWorklist(NOTNode.
getNode());
8235 if (SimplifySelectOps(
N, N1, N2))
8247 bool normalizeToSequence =
8256 if (normalizeToSequence || !InnerSelect.
use_empty())
8258 InnerSelect, N2, Flags);
8261 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
8268 Cond1, N1, N2, Flags);
8269 if (normalizeToSequence || !InnerSelect.
use_empty())
8271 InnerSelect, Flags);
8274 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
8284 if (!normalizeToSequence) {
8290 if (
SDValue Combined = visitANDLike(N0, N1_0,
N)) {
8303 if (!normalizeToSequence) {
8309 if (
SDValue Combined = visitORLike(N0, N2_0,
N))
8345 auto *
C = dyn_cast<ConstantSDNode>(N2.
getOperand(1));
8346 auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
8347 if (
C && NotC &&
C->getAPIntValue() == ~NotC->getAPIntValue()) {
8367 (!LegalOperations &&
8378 return SimplifySelect(DL, N0, N1, N2);
8391 SDValue Lo, Hi, LL, LH, RL, RH;
8395 Lo = DAG.
getNode(
N->getOpcode(), DL, LoVT, LL, RL,
N->getOperand(2));
8396 Hi = DAG.
getNode(
N->getOpcode(), DL, HiVT, LH, RH,
N->getOperand(2));
8398 return std::make_pair(Lo, Hi);
8408 EVT VT =
N->getValueType(0);
8425 for (
int i = 0; i < NumElems / 2; ++i) {
8426 if (Cond->getOperand(i)->isUndef())
8429 if (BottomHalf ==
nullptr)
8430 BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
8431 else if (Cond->getOperand(i).getNode() != BottomHalf)
8437 for (
int i = NumElems / 2; i < NumElems; ++i) {
8438 if (Cond->getOperand(i)->isUndef())
8441 if (TopHalf ==
nullptr)
8442 TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
8443 else if (Cond->getOperand(i).getNode() != TopHalf)
8447 assert(TopHalf && BottomHalf &&
8448 "One half of the selector was all UNDEFs and the other was all the "
8449 "same value. This should have been addressed before this function.");
8490 EVT LoMemVT, HiMemVT;
8506 SDValue OpsLo[] = { Chain, DataLo, MaskLo,
BasePtr, IndexLo, Scale };
8551 EVT LoMemVT, HiMemVT;
8579 AddToWorklist(
Lo.getNode());
8580 AddToWorklist(
Hi.getNode());
8607 EVT VT =
N->getValueType(0);
8618 SDValue PassThruLo, PassThruHi;
8619 std::tie(PassThruLo, PassThruHi) = DAG.
SplitVector(PassThru, DL);
8628 EVT LoMemVT, HiMemVT;
8635 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, DL);
8642 SDValue OpsLo[] = { Chain, PassThruLo, MaskLo,
BasePtr, IndexLo, Scale };
8646 SDValue OpsHi[] = { Chain, PassThruHi, MaskHi,
BasePtr, IndexHi, Scale };
8650 AddToWorklist(
Lo.getNode());
8651 AddToWorklist(
Hi.getNode());
8664 SDValue RetOps[] = { GatherRes, Chain };
8685 EVT VT =
N->getValueType(0);
8696 SDValue PassThruLo, PassThruHi;
8697 std::tie(PassThruLo, PassThruHi) = DAG.
SplitVector(PassThru, DL);
8707 EVT LoMemVT, HiMemVT;
8715 Lo = DAG.
getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, PassThruLo, LoMemVT,
8727 Hi = DAG.
getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, PassThruHi, HiMemVT,
8730 AddToWorklist(
Lo.getNode());
8731 AddToWorklist(
Hi.getNode());
8744 SDValue RetOps[] = { LoadRes, Chain };
8756 EVT VT =
N->getValueType(0);
8757 if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
8766 bool AllAddOne =
true;
8767 bool AllSubOne =
true;
8769 for (
unsigned i = 0; i != Elts; ++i) {
8775 const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
8776 const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
8786 if (AllAddOne || AllSubOne) {
8805 EVT VT =
N->getValueType(0);
8843 AddToWorklist(Shift.
getNode());
8844 AddToWorklist(
Add.getNode());
8875 SetCCWidth != 1 && SetCCWidth < WideWidth &&
8886 SDValue WideSetCC = DAG.
getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
8892 if (SimplifySelectOps(
N, N1, N2))
8912 if (
SDValue V = foldVSelectOfConstants(
N))
8933 AddToWorklist(SCC.getNode());
8935 if (
ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
8936 if (!SCCC->isNullValue())
8940 }
else if (SCC->isUndef()) {
8948 SCC.getOperand(1), N2, N3, SCC.getOperand(2));
8949 SelectOp->
setFlags(SCC->getFlags());
8955 if (SimplifySelectOps(
N, N2, N3))
8959 return SimplifySelectCC(
SDLoc(
N), N0, N1, N2, N3, CC);
8967 N->hasOneUse() &&
N->use_begin()->getOpcode() ==
ISD::BRCOND;
8970 N->getValueType(0),
N->getOperand(0),
N->getOperand(1),
8971 cast<CondCodeSDNode>(
N->getOperand(2))->get(),
SDLoc(
N), !PreferSetCC);
8979 SDValue NewSetCC = rebuildSetCC(Combined);
9013 unsigned Opcode =
N->getOpcode();
9015 EVT VT =
N->getValueType(0);
9021 &&
"Expected EXTEND dag node in input!");
9026 if (isa<ConstantSDNode>(N0))
9027 return DAG.
getNode(Opcode, DL, VT, N0);
9035 if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
9046 unsigned FoldOpc = Opcode;
9050 DAG.
getNode(FoldOpc, DL, VT, Op1),
9051 DAG.
getNode(FoldOpc, DL, VT, Op2));
9074 for (
unsigned i = 0; i != NumElts; ++i) {
9084 APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
9102 bool HasCopyToRegUses =
false;
9110 if (UI.getUse().getResNo() != N0.
getResNo())
9119 for (
unsigned i = 0; i != 2; ++i) {
9123 if (!isa<ConstantSDNode>(UseOp))
9137 HasCopyToRegUses =
true;
9140 if (HasCopyToRegUses) {
9141 bool BothLiveOut =
false;
9153 return ExtendNodes.
size();
9163 for (
SDNode *SetCC : SetCCs) {
9166 for (
unsigned j = 0; j != 2; ++j) {
9168 if (SOp == OrigLoad)
9182 EVT DstVT =
N->getValueType(0);
9187 "Unexpected node type (not an extend)!");
9224 EVT SplitSrcVT = SrcVT;
9225 EVT SplitDstVT = DstVT;
9236 const unsigned NumSplits =
9243 for (
unsigned Idx = 0;
Idx < NumSplits;
Idx++) {
9263 AddToWorklist(NewChain.
getNode());
9265 CombineTo(
N, NewValue);
9271 ExtendSetCCUses(SetCCs, N0, NewValue, (
ISD::NodeType)
N->getOpcode());
9272 CombineTo(N0.
getNode(), Trunc, NewChain);
9280 EVT VT =
N->getValueType(0);
9281 EVT OrigVT =
N->getOperand(0).getValueType();
9304 EVT MemVT =
Load->getMemoryVT();
9325 Load->getChain(),
Load->getBasePtr(),
9326 Load->getMemoryVT(),
Load->getMemOperand());
9344 Load->getValueType(0), ExtLoad);
9345 CombineTo(Load, Trunc, ExtLoad.
getValue(1));
9349 recursivelyDeleteUnusedNodes(N0.
getNode());
9358SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(
SDNode *Cast) {
9359 unsigned CastOpcode = Cast->
getOpcode();
9363 "Unexpected opcode for vector select narrowing/widening");
9393 CastA = DAG.
getNode(CastOpcode, DL, VT,
A);
9394 CastB = DAG.
getNode(CastOpcode, DL, VT,
B);
9403 bool LegalOperations,
SDNode *
N,
9424 Combiner.recursivelyDeleteUnusedNodes(LN0);
9438 ((LegalOperations || VT.
isVector() ||
9439 cast<LoadSDNode>(N0)->isVolatile()) &&
9443 bool DoXform =
true;
9456 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
9460 if (NoReplaceTrunc) {
9462 Combiner.recursivelyDeleteUnusedNodes(LN0);
9472 bool LegalOperations) {
9484 EVT VT =
N->getValueType(0);
9485 EVT XVT =
X.getValueType();
9497 return DAG.
getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount);
9504 EVT VT =
N->getValueType(0);
9520 if (NarrowLoad.getNode() != N0.
getNode()) {
9521 CombineTo(N0.
getNode(), NarrowLoad);
9531 unsigned OpBits =
Op.getScalarValueSizeInBits();
9536 if (OpBits == DestBits) {
9539 if (NumSignBits > DestBits-MidBits)
9541 }
else if (OpBits < DestBits) {
9544 if (NumSignBits > OpBits-MidBits)
9549 if (NumSignBits > OpBits-MidBits)
9556 if (OpBits < DestBits)
9558 else if (OpBits > DestBits)
9573 if (
SDValue ExtLoad = CombineExtLoad(
N))
9605 bool NoReplaceTruncAnd = !N0.
hasOneUse();
9609 if (NoReplaceTruncAnd) {
9612 CombineTo(N0.
getNode(), TruncAnd);
9614 if (NoReplaceTrunc) {
9619 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
9637 if (VT.
isVector() && !LegalOperations &&
9653 return DAG.
getSetCC(DL, VT, N00, N01, CC);
9659 if (SVT == MatchingVecType) {
9678 SDValue ExtTrueVal = (SetCCWidth == 1)
9683 SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC,
true))
9695 return DAG.
getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
9705 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
9737 Op =
N->getOperand(0);
9743 N.getValueType().getScalarType() !=
MVT::i1 ||
9744 cast<CondCodeSDNode>(
N.getOperand(2))->get() !=
ISD::SETNE)
9760 return (Known.
Zero | 1).isAllOnesValue();
9765 EVT VT =
N->getValueType(0);
9782 APInt TruncatedBits =
9784 APInt(
Op.getScalarValueSizeInBits(), 0) :
9787 std::min(
Op.getScalarValueSizeInBits(),
9799 if (NarrowLoad.getNode() != N0.
getNode()) {
9800 CombineTo(N0.
getNode(), NarrowLoad);
9817 AddToWorklist(
Op.getNode());
9827 AddToWorklist(
Op.getNode());
9861 if (
SDValue ExtLoad = CombineExtLoad(
N))
9877 bool DoXform =
true;
9881 auto *AndC = cast<ConstantSDNode>(N0.
getOperand(1));
9884 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
9902 bool NoReplaceTruncAnd = !N0.
hasOneUse();
9906 if (NoReplaceTruncAnd) {
9909 CombineTo(N0.
getNode(), TruncAnd);
9911 if (NoReplaceTrunc) {
9916 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
9925 if (
SDValue ZExtLoad = CombineZExtLogicopShiftLoad(
N))
9938 if (!LegalOperations && VT.
isVector() &&
9971 if (
SDValue SCC = SimplifySelectCC(
9974 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
9990 if (cast<ConstantSDNode>(ShAmt)->getAPIntValue().ugt(KnownZeroBits))
10005 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
10013 EVT VT =
N->getValueType(0);
10031 if (NarrowLoad.getNode() != N0.
getNode()) {
10032 CombineTo(N0.
getNode(), NarrowLoad);
10034 AddToWorklist(oye);
10067 bool DoXform =
true;
10081 CombineTo(
N, ExtLoad);
10082 if (NoReplaceTrunc) {
10084 recursivelyDeleteUnusedNodes(LN0);
10088 CombineTo(LN0, Trunc, ExtLoad.
getValue(1));
10102 if (!LegalOperations || TLI.
isLoadExtLegal(ExtType, VT, MemVT)) {
10106 CombineTo(
N, ExtLoad);
10108 recursivelyDeleteUnusedNodes(LN0);
10119 if (VT.
isVector() && !LegalOperations) {
10132 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
10141 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
10147 if (
SDValue SCC = SimplifySelectCC(
10150 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
10158 unsigned Opcode =
N->getOpcode();
10161 EVT AssertVT = cast<VTSDNode>(N1)->getVT();
10165 AssertVT == cast<VTSDNode>(N0.
getOperand(1))->getVT())
10176 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
10178 "Asserting zero/sign-extended bits to a type larger than the "
10179 "truncated destination does not provide information");
10182 EVT MinAssertVT = AssertVT.
bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
10196 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
10198 "Asserting zero/sign-extended bits to a type larger than the "
10199 "truncated destination does not provide information");
10201 if (AssertVT.
bitsLT(BigA_AssertVT)) {
10219 unsigned Opc =
N->getOpcode();
10223 EVT VT =
N->getValueType(0);
10230 unsigned ShAmt = 0;
10231 bool HasShiftedOffset =
false;
10236 ExtVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
10244 auto *LN0 = dyn_cast<LoadSDNode>(N0.
getOperand(0));
10245 auto *N01 = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
10249 uint64_t ShiftAmt = N01->getZExtValue();
10258 auto AndC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
10263 unsigned ActiveBits = 0;
10264 if (
Mask.isMask()) {
10265 ActiveBits =
Mask.countTrailingOnes();
10266 }
else if (
Mask.isShiftedMask()) {
10267 ShAmt =
Mask.countTrailingZeros();
10270 HasShiftedOffset =
true;
10280 if (
auto *ConstShift = dyn_cast<ConstantSDNode>(
SRL.getOperand(1))) {
10281 ShAmt = ConstShift->getZExtValue();
10284 if ((ShAmt & (EVTBits-1)) == 0) {
10292 if (!isa<LoadSDNode>(N0))
return SDValue();
10294 auto *LN0 = cast<LoadSDNode>(N0);
10312 isa<ConstantSDNode>(
Mask->getOperand(1))) {
10313 const APInt &ShiftMask =
10314 cast<ConstantSDNode>(
Mask->getOperand(1))->getAPIntValue();
10315 if (ShiftMask.
isMask()) {
10329 unsigned ShLeftAmt = 0;
10333 ShLeftAmt = N01->getZExtValue();
10339 if (!isa<LoadSDNode>(N0))
10343 if (!isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
10346 auto AdjustBigEndianShift = [&](
unsigned ShAmt) {
10349 return LVTStoreBits - EVTStoreBits - ShAmt;
10355 ShAmt = AdjustBigEndianShift(ShAmt);
10358 uint64_t PtrOff = ShAmt / 8;
10368 AddToWorklist(NewPtr.
getNode());
10382 WorklistRemover DeadNodes(*
this);
10387 if (ShLeftAmt != 0) {
10400 Result, DAG.
getConstant(ShLeftAmt, DL, ShImmTy));
10403 if (HasShiftedOffset) {
10407 ShAmt = AdjustBigEndianShift(ShAmt);
10425 EVT VT =
N->getValueType(0);
10426 EVT EVT = cast<VTSDNode>(N1)->getVT();
10454 if ((N00Bits <= EVTBits ||
10465 if (!LegalOperations ||
10491 if (
SDValue NarrowLoad = ReduceLoadWidth(
N))
10498 if (
auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1)))
10499 if (ShAmt->getAPIntValue().ule(VTBits - EVTBits)) {
10503 if (((VTBits - EVTBits) - ShAmt->getZExtValue()) < InSignBits)
10515 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
10516 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile() &&
10524 CombineTo(
N, ExtLoad);
10526 AddToWorklist(ExtLoad.
getNode());
10532 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
10533 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
10540 CombineTo(
N, ExtLoad);
10558 EVT VT =
N->getValueType(0);
10574 EVT VT =
N->getValueType(0);
10590 EVT VT =
N->getValueType(0);
10605 if (
C.getNode() !=
N)
10642 EVT TrTy =
N->getValueType(0);
10651 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
10652 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
10654 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
10689 AddToWorklist(Amt.
getNode());
10721 if (BuildVectEltTy == TruncVecEltTy) {
10725 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
10727 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
10728 "Invalid number of elements");
10731 for (
unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset)
10753 if (
SDValue Reduced = ReduceLoadWidth(
N))
10778 unsigned NumDefs = 0;
10782 if (!
X.isUndef()) {
10792 X.getValueType().getVectorNumElements()));
10798 if (NumDefs == 1) {
10799 assert(V.
getNode() &&
"The single defined operand is empty!");
10801 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
10807 AddToWorklist(
NV.getNode());
10822 (!LegalOperations ||
10869 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
10883 if (!LegalOperations && N0.
hasOneUse() &&
10933 if (NewAlign <= Align &&
10952 EVT VT =
N->getValueType(0);
10992 NumFPLogicOpsConv++;
11003 EVT VT =
N->getValueType(0);
11019 cast<BuildVectorSDNode>(N0)->isConstant())
11020 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(),
11024 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
11028 if (!LegalOperations ||
11034 if (
C.getNode() !=
N)
11054 ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
11089 AddToWorklist(NewConv.
getNode());
11099 AddToWorklist(FlipBit.
getNode());
11106 AddToWorklist(
Hi.getNode());
11108 AddToWorklist(FlipBit.
getNode());
11112 AddToWorklist(FlipBits.
getNode());
11142 AddToWorklist(
X.getNode());
11146 if (OrigXWidth < VTWidth) {
11148 AddToWorklist(
X.getNode());
11149 }
else if (OrigXWidth > VTWidth) {
11154 X.getValueType(),
X,
11156 X.getValueType()));
11157 AddToWorklist(
X.getNode());
11159 AddToWorklist(
X.getNode());
11165 AddToWorklist(Cst.
getNode());
11167 AddToWorklist(
X.getNode());
11169 AddToWorklist(XorResult.
getNode());
11173 SDLoc(XorResult)));
11174 AddToWorklist(XorResult64.
getNode());
11178 AddToWorklist(FlipBit.
getNode());
11181 AddToWorklist(FlipBits.
getNode());
11187 AddToWorklist(
X.getNode());
11192 AddToWorklist(Cst.
getNode());
11200 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
11215 auto PeekThroughBitcast = [&](
SDValue Op) {
11217 Op.getOperand(0).getValueType() == VT)
11237 for (
int i = 0; i != MaskScale; ++i)
11238 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
11255 EVT VT =
N->getValueType(0);
11256 return CombineConsecutiveLoads(
N, VT);
11262ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV,
EVT DstEltVT) {
11266 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
11273 if (SrcBitSize == DstBitSize) {
11278 if (
Op.getValueType() != SrcEltVT)
11281 AddToWorklist(Ops.
back().getNode());
11295 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
11303 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
11306 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
11314 if (SrcBitSize < DstBitSize) {
11315 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
11319 i += NumInputsPerOutput) {
11322 bool EltIsUndef =
true;
11323 for (
unsigned j = 0; j != NumInputsPerOutput; ++j) {
11325 NewBits <<= SrcBitSize;
11327 if (
Op.isUndef())
continue;
11328 EltIsUndef =
false;
11330 NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue().
11331 zextOrTrunc(SrcBitSize).
zext(DstBitSize);
11346 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
11352 if (
Op.isUndef()) {
11357 APInt OpVal = cast<ConstantSDNode>(Op)->
11360 for (
unsigned j = 0; j != NumOutputsPerInput; ++j) {
11368 std::reverse(Ops.
end()-NumOutputsPerInput, Ops.
end());
11376 return F.hasAllowContract() ||
F.hasAllowReassociation();
11383 EVT VT =
N->getValueType(0);
11397 if (!HasFMAD && !HasFMA)
11403 CanFuse || HasFMAD);
11418 auto isContractableFMUL = [AllowFusionGlobally](
SDValue N) {
11425 if (
Aggressive && isContractableFMUL(N0) && isContractableFMUL(N1)) {
11432 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11439 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11448 if (isContractableFMUL(N00) &&
11450 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11462 if (isContractableFMUL(N10) &&
11464 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11476 N0.
getOpcode() == PreferredFusedOpcode &&
11479 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11481 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11484 N1, Flags), Flags);
11489 N1->
getOpcode() == PreferredFusedOpcode &&
11492 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11494 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11497 N0, Flags), Flags);
11503 auto FoldFAddFMAFPExtFMul = [&] (
11506 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X,
Y,
11507 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11512 if (N0.
getOpcode() == PreferredFusedOpcode) {
11516 if (isContractableFMUL(N020) &&
11530 auto FoldFAddFPExtFMAFMul = [&] (
11533 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11536 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11543 if (N00.
getOpcode() == PreferredFusedOpcode) {
11545 if (isContractableFMUL(N002) &&
11556 if (N1.
getOpcode() == PreferredFusedOpcode) {
11560 if (isContractableFMUL(N120) &&
11576 if (N10.
getOpcode() == PreferredFusedOpcode) {
11578 if (isContractableFMUL(N102) &&
11595 EVT VT =
N->getValueType(0);
11608 if (!HasFMAD && !HasFMA)
11614 CanFuse || HasFMAD);
11630 auto isContractableFMUL = [AllowFusionGlobally](
SDValue N) {
11638 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11646 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11657 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11668 if (isContractableFMUL(N00) &&
11670 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11684 if (isContractableFMUL(N10) &&
11686 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11706 if (isContractableFMUL(N000) &&
11709 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11729 if (isContractableFMUL(N000) &&
11732 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11746 if (CanFuse && N0.
getOpcode() == PreferredFusedOpcode &&
11749 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11751 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11755 N1), Flags), Flags);
11760 if (CanFuse && N1.
getOpcode() == PreferredFusedOpcode &&
11764 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11768 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11770 N21, N0, Flags), Flags);
11776 if (N0.
getOpcode() == PreferredFusedOpcode) {
11780 if (isContractableFMUL(N020) &&
11782 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11784 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11790 N1), Flags), Flags);
11803 if (N00.
getOpcode() == PreferredFusedOpcode) {
11805 if (isContractableFMUL(N002) &&
11807 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11812 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11818 N1), Flags), Flags);
11825 if (N1.
getOpcode() == PreferredFusedOpcode &&
11828 if (isContractableFMUL(N120) &&
11832 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11835 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11841 N0, Flags), Flags);
11857 if (isContractableFMUL(N102) &&
11861 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11866 DAG.
getNode(PreferredFusedOpcode, SL, VT,
11872 N0, Flags), Flags);
11883SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *
N) {
11886 EVT VT =
N->getValueType(0);
11911 if (!HasFMAD && !HasFMA)
11923 if (
C->isExactlyValue(+1.0))
11924 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
11926 if (
C->isExactlyValue(-1.0))
11927 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
11934 if (
SDValue FMA = FuseFADD(N0, N1, Flags))
11936 if (
SDValue FMA = FuseFADD(N1, N0, Flags))
11946 if (C0->isExactlyValue(+1.0))
11947 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11950 if (C0->isExactlyValue(-1.0))
11951 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
11956 if (C1->isExactlyValue(+1.0))
11957 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
11959 if (C1->isExactlyValue(-1.0))
11960 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
11967 if (
SDValue FMA = FuseFSUB(N0, N1, Flags))
11969 if (
SDValue FMA = FuseFSUB(N1, N0, Flags))
11980 EVT VT =
N->getValueType(0);
11987 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
11991 if (N0CFP && N1CFP)
11995 if (N0CFP && !N1CFP)
12000 if (N1C && N1C->
isZero())
12004 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
12012 ForCodeSize), Flags);
12019 ForCodeSize), Flags);
12021 auto isFMulNegTwo = [](
SDValue FMul) {
12022 if (!FMul.hasOneUse() || FMul.getOpcode() !=
ISD::FMUL)
12025 return C &&
C->isExactlyValue(-2.0);
12029 if (isFMulNegTwo(N0)) {
12035 if (isFMulNegTwo(N1)) {
12078 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
12099 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
12147 if (
SDValue Fused = visitFADDForFMACombine(
N)) {
12148 AddToWorklist(Fused.getNode());
12159 EVT VT =
N->getValueType(0);
12166 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
12170 if (N0CFP && N1CFP)
12173 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
12177 if (N1CFP && N1CFP->
isZero()) {
12195 if (N0CFP && N0CFP->
isZero()) {
12220 ForCodeSize), Flags);
12223 if (
SDValue Fused = visitFSUBForFMACombine(
N)) {
12224 AddToWorklist(Fused.getNode());
12236 EVT VT =
N->getValueType(0);
12244 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
12249 if (N0CFP && N1CFP)
12261 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
12267 if (N1CFP && N1CFP->
isZero())
12312 if (LHSNeg == 2 || RHSNeg == 2)
12332 auto TrueOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(1));
12333 auto FalseOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(2));
12335 if (TrueOpnd && FalseOpnd &&
12336 Cond.getOpcode() ==
ISD::SETCC && Cond.getOperand(0) ==
X &&
12337 isa<ConstantFPSDNode>(Cond.getOperand(1)) &&
12338 cast<ConstantFPSDNode>(Cond.getOperand(1))->isExactlyValue(0.0)) {
12339 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
12356 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
12360 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
12369 if (
SDValue Fused = visitFMULForFMADistributiveCombine(
N)) {
12370 AddToWorklist(Fused.getNode());
12383 EVT VT =
N->getValueType(0);
12392 if (isa<ConstantFPSDNode>(N0) &&
12393 isa<ConstantFPSDNode>(N1) &&
12394 isa<ConstantFPSDNode>(N2)) {
12398 if (UnsafeFPMath) {
12399 if (N0CFP && N0CFP->
isZero())
12401 if (N1CFP && N1CFP->
isZero())
12415 if (UnsafeFPMath) {
12447 AddToWorklist(RHSNeg.
getNode());
12462 if (UnsafeFPMath) {
12464 if (N1CFP && N0 == N2) {
12512 unsigned NumElts = 1;
12513 EVT VT =
N->getValueType(0);
12517 if (!MinUses || (N1->
use_size() * NumElts) < MinUses)
12523 for (
auto *U : N1->
uses()) {
12524 if (U->getOpcode() ==
ISD::FDIV && U->getOperand(1) == N1) {
12527 if (UnsafeMath || U->getFlags().hasAllowReciprocal())
12534 if ((
Users.size() * NumElts) < MinUses)
12542 for (
auto *U :
Users) {
12544 if (Dividend != FPOne) {
12546 Reciprocal, Flags);
12547 CombineTo(U, NewNode);
12548 }
else if (U != Reciprocal.
getNode()) {
12551 CombineTo(U, Reciprocal);
12562 EVT VT =
N->getValueType(0);
12569 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
12573 if (N0CFP && N1CFP)
12576 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
12592 (!LegalOperations ||
12648 if (
SDValue RV = BuildReciprocalEstimate(N1, Flags)) {
12661 if (LHSNeg == 2 || RHSNeg == 2)
12679 EVT VT =
N->getValueType(0);
12682 if (N0CFP && N1CFP)
12685 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
12702 return buildSqrtEstimate(N0, Flags);
12717 return (N1VT == N1Op0VT || N1Op0VT !=
MVT::f128);
12727 EVT VT =
N->getValueType(0);
12729 if (N0CFP && N1CFP)
12778 EVT VT =
N->getValueType(0);
12808 if (ExponentIs025 || ExponentIs075) {
12851 Attribute StrictOverflow =
F.getFnAttribute(
"strict-float-cast-overflow");
12861 EVT VT =
N->getValueType(0);
12882 EVT VT =
N->getValueType(0);
12892 (!LegalOperations ||
12910 (!LegalOperations ||
12924 (!LegalOperations ||
12943 EVT VT =
N->getValueType(0);
12953 (!LegalOperations ||
12970 (!LegalOperations ||
12990 EVT VT =
N->getValueType(0);
13011 unsigned ActualSize = std::min(InputSize, OutputSize);
13031 EVT VT =
N->getValueType(0);
13046 EVT VT =
N->getValueType(0);
13063 EVT VT =
N->getValueType(0);
13075 const bool NIsTrunc =
N->getConstantOperandVal(1) == 1;
13104 AddToWorklist(Tmp.
getNode());
13109 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13117 EVT VT =
N->getValueType(0);
13118 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
13133 EVT VT =
N->getValueType(0);
13136 if (
N->hasOneUse() &&
13154 if (
In.getValueType() == VT)
return In;
13169 CombineTo(
N, ExtLoad);
13178 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13186 EVT VT =
N->getValueType(0);
13197 EVT VT =
N->getValueType(0);
13221 EVT VT =
N->getValueType(0);
13233 EVT VT =
N->getValueType(0);
13249 EVT IntVT =
Int.getValueType();
13264 AddToWorklist(
Int.getNode());
13293 EVT VT =
N->getValueType(0);
13297 if (N0CFP && N1CFP) {
13329 EVT VT =
N->getValueType(0);
13337 return N->getOperand(0);
13347 EVT IntVT =
Int.getValueType();
13362 AddToWorklist(
Int.getNode());
13392 if (
SDValue NewN1 = rebuildSetCC(N1))
13402 (
N.getOperand(0).hasOneUse() &&
13403 N.getOperand(0).getOpcode() ==
ISD::SRL))) {
13406 N =
N.getOperand(0);
13432 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
13435 cast<ConstantSDNode>(Op1)->getAPIntValue() == AndConst.
logBase2()) {
13455 SDValue Tmp = visitXOR(
N.getNode());
13462 N = XORHandle.getValue();
13470 SDNode *TheXor =
N.getNode();
13476 bool Equal =
false;
13483 EVT SetCCVT =
N.getValueType();
13499 SDValue CondLHS =
N->getOperand(2), CondRHS =
N->getOperand(3);
13532 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
13534 VT = LD->getMemoryVT();
13535 AS = LD->getAddressSpace();
13537 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
13539 VT = ST->getMemoryVT();
13540 AS = ST->getAddressSpace();
13550 AM.
BaseOffs = Offset->getSExtValue();
13554 }
else if (
N->getOpcode() ==
ISD::SUB) {
13559 AM.
BaseOffs = -Offset->getSExtValue();
13575bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *
N) {
13583 if (
LD->isIndexed())
13585 VT =
LD->getMemoryVT();
13589 Ptr =
LD->getBasePtr();
13590 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
13591 if (
ST->isIndexed())
13593 VT =
ST->getMemoryVT();
13597 Ptr =
ST->getBasePtr();
13619 bool Swapped =
false;
13620 if (isa<ConstantSDNode>(BasePtr)) {
13639 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
13644 SDValue Val = cast<StoreSDNode>(
N)->getValue();
13647 if (Val == BasePtr)
13664 if (isa<ConstantSDNode>(Offset))
13666 UE =
BasePtr.getNode()->use_end();
13684 if (!isa<ConstantSDNode>(Op1)) {
13702 bool RealUse =
false;
13722 BasePtr, Offset, AM);
13725 BasePtr, Offset, AM);
13729 Result.getNode()->dump(&DAG);
dbgs() <<
'\n');
13730 WorklistRemover DeadNodes(*
this);
13739 deleteAndRecombine(
N);
13745 for (
unsigned i = 0, e = OtherUses.
size(); i != e; ++i) {
13746 unsigned OffsetIdx = 1;
13747 if (OtherUses[i]->getOperand(OffsetIdx).getNode() ==
BasePtr.getNode())
13749 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
13750 BasePtr.getNode() &&
"Expected BasePtr operand");
13764 cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx));
13765 int X0, X1, Y0, Y1;
13767 APInt Offset1 = cast<ConstantSDNode>(Offset)->getAPIntValue();
13769 X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
13770 Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
13776 APInt CNV = Offset0;
13777 if (X0 < 0) CNV = -CNV;
13778 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
13779 else CNV = CNV - Offset1;
13781 SDLoc DL(OtherUses[i]);
13791 deleteAndRecombine(OtherUses[i]);
13796 deleteAndRecombine(Ptr.
getNode());
13797 AddToWorklist(
Result.getNode());
13806bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *
N) {
13814 if (
LD->isIndexed())
13816 VT =
LD->getMemoryVT();
13820 Ptr =
LD->getBasePtr();
13821 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
13822 if (
ST->isIndexed())
13824 VT =
ST->getMemoryVT();
13828 Ptr =
ST->getBasePtr();
13857 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
13861 bool TryNext =
false;
13869 bool RealUse =
false;
13896 BasePtr, Offset, AM)
13898 BasePtr, Offset, AM);
13899 ++PostIndexedNodes;
13902 dbgs() <<
"\nWith: ";
Result.getNode()->dump(&DAG);
13904 WorklistRemover DeadNodes(*
this);
13913 deleteAndRecombine(
N);
13918 deleteAndRecombine(Op);
13938 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
13939 "Cannot split out indexing using opaque target constants");
13952 return T.isVector() ?
T.getVectorNumElements() : 0;
13956 Val =
ST->getValue();
13958 EVT STMemType =
ST->getMemoryVT();
13959 if (STType == STMemType)
13981 EVT LDMemType =
LD->getMemoryVT();
13982 EVT LDType =
LD->getValueType(0);
13984 "Attempting to extend value of non-matching type");
13985 if (LDType == LDMemType)
13988 switch (
LD->getExtensionType()) {
14011 if (!ST ||
ST->isVolatile())
14014 EVT LDType =
LD->getValueType(0);
14015 EVT LDMemType =
LD->getMemoryVT();
14016 EVT STMemType =
ST->getMemoryVT();
14017 EVT STType =
ST->getValue().getValueType();
14039 if (
LD->isIndexed()) {
14044 LD->getOperand(1),
LD->getOperand(2));
14046 return CombineTo(LD, Ops, 3);
14048 return CombineTo(LD, Val, Chain);
14055 if (Offset == 0 && LDType == STType && STMemType == LDMemType) {
14058 return ReplaceLd(LD,
ST->getValue(), Chain);
14066 SDLoc(ST), STType);
14068 return ReplaceLd(LD, Val, Chain);
14073 if (
LD->getBasePtr().isUndef() || Offset != 0)
14079 if (!getTruncatedStoreValue(ST, Val))
14083 if (STMemType != LDMemType) {
14091 if (!extendLoadedValueToExtension(LD, Val))
14093 return ReplaceLd(LD, Val, Chain);
14098 deleteAndRecombine(Val.
getNode());
14110 if (!
LD->isVolatile()) {
14113 if (!
N->hasAnyUseOfValue(0)) {
14123 WorklistRemover DeadNodes(*
this);
14125 AddUsersToWorklist(Chain.
getNode());
14126 if (
N->use_empty())
14127 deleteAndRecombine(
N);
14140 cast<ConstantSDNode>(
LD->getOperand(2))->isOpaque();
14142 if (!
N->hasAnyUseOfValue(0) &&
14147 Index = SplitIndexingFromLoad(LD);
14150 AddUsersToWorklist(
N);
14154 dbgs() <<
"\nWith: ";
Undef.getNode()->dump(&DAG);
14155 dbgs() <<
" and 2 other values\n");
14156 WorklistRemover DeadNodes(*
this);
14160 deleteAndRecombine(
N);
14168 if (
auto V = ForwardStoreValueToDirectLoad(LD))
14174 if (Align >
LD->getAlignment() &&
LD->getSrcValueOffset() % Align == 0) {
14176 LD->getExtensionType(),
SDLoc(
N),
LD->getValueType(0), Chain, Ptr,
14177 LD->getPointerInfo(),
LD->getMemoryVT(), Align,
14178 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
14186 if (
LD->isUnindexed()) {
14188 SDValue BetterChain = FindBetterChain(LD, Chain);
14191 if (Chain != BetterChain) {
14197 BetterChain, Ptr,
LD->getMemOperand());
14200 LD->getValueType(0),
14201 BetterChain, Ptr,
LD->getMemoryVT(),
14202 LD->getMemOperand());
14210 return CombineTo(
N, ReplLoad.
getValue(0), Token);
14215 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
14220 if (SliceUpLoad(
N))
14240struct LoadedSlice {
14247 unsigned Loads = 0;
14248 unsigned Truncates = 0;
14249 unsigned CrossRegisterBanksCopies = 0;
14250 unsigned ZExts = 0;
14251 unsigned Shift = 0;
14253 Cost(
bool ForCodeSize =
false) : ForCodeSize(ForCodeSize) {}
14256 Cost(
const LoadedSlice &LS,
bool ForCodeSize =
false)
14257 : ForCodeSize(ForCodeSize), Loads(1) {
14258 EVT TruncType =
LS.Inst->getValueType(0);
14259 EVT LoadedType =
LS.getLoadedType();
14260 if (TruncType != LoadedType &&
14261 !
LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
14269 void addSliceGain(
const LoadedSlice &LS) {
14273 LS.Inst->getValueType(0)))
14279 if (
LS.canMergeExpensiveCrossRegisterBankCopy())
14280 ++CrossRegisterBanksCopies;
14284 Loads += RHS.Loads;
14285 Truncates += RHS.Truncates;
14286 CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies;
14287 ZExts += RHS.ZExts;
14288 Shift += RHS.Shift;
14293 return Loads == RHS.Loads && Truncates == RHS.Truncates &&
14294 CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies &&
14295 ZExts == RHS.ZExts && Shift == RHS.Shift;
14298 bool operator!=(
const Cost &RHS)
const {
return !(*
this == RHS); }
14300 bool operator<(
const Cost &RHS)
const {
14303 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
14304 unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies;
14307 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
14308 return ExpensiveOpsLHS < ExpensiveOpsRHS;
14309 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
14310 (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS);
14313 bool operator>(
const Cost &RHS)
const {
return RHS < *
this; }
14315 bool operator<=(
const Cost &RHS)
const {
return !(RHS < *
this); }
14317 bool operator>=(
const Cost &RHS)
const {
return !(*
this < RHS); }
14336 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
14341 APInt getUsedBits()
const {
14346 assert(Origin &&
"No original load to compare against.");
14348 assert(Inst &&
"This slice is not bound to an instruction");
14350 "Extracted slice is bigger than the whole type!");
14352 UsedBits.setAllBits();
14353 UsedBits = UsedBits.zext(BitWidth);
14354 UsedBits <<= Shift;
14359 unsigned getLoadedSize()
const {
14360 unsigned SliceSize = getUsedBits().countPopulation();
14361 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
14362 return SliceSize / 8;
14367 EVT getLoadedType()
const {
14368 assert(DAG &&
"Missing context");
14376 uint64_t
Offset = getOffsetFromBase();
14378 Alignment =
MinAlign(Alignment, Alignment + Offset);
14383 bool isLegal()
const {
14385 if (!Origin || !Inst || !DAG)
14395 EVT SliceType = getLoadedType();
14419 if (TruncateType != SliceType &&
14429 uint64_t getOffsetFromBase()
const {
14430 assert(DAG &&
"Missing context.");
14432 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
14433 uint64_t
Offset = Shift / 8;
14436 "The size of the original loaded type is not a multiple of a"
14440 assert(TySizeInBytes > Offset &&
14441 "Invalid shift amount for given loaded size");
14454 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
14456 SDValue BaseAddr = OldBaseAddr;
14458 int64_t
Offset =
static_cast<int64_t
>(getOffsetFromBase());
14459 assert(Offset >= 0 &&
"Offset too big to fit in int64_t!");
14469 EVT SliceType = getLoadedType();
14479 if (SliceType != FinalType)
14489 bool canMergeExpensiveCrossRegisterBankCopy()
const {
14495 assert(DAG &&
"Missing context");
14497 EVT ResVT =
Use->getValueType(0);
14502 Use->getOperand(0)->isDivergent());
14511 if (!
TRI ||
TRI->getCommonSubClass(ArgRC, ResRC))
14556 const LoadedSlice &Second) {
14557 assert(First.Origin == Second.Origin && First.Origin &&
14558 "Unable to match different memory origins.");
14559 APInt UsedBits = First.getUsedBits();
14560 assert((UsedBits & Second.getUsedBits()) == 0 &&
14561 "Slices are not supposed to overlap.");
14562 UsedBits |= Second.getUsedBits();
14571 LoadedSlice::Cost &GlobalLSCost) {
14572 unsigned NumberOfSlices = LoadedSlices.
size();
14574 if (NumberOfSlices < 2)
14579 llvm::sort(LoadedSlices, [](
const LoadedSlice &LHS,
const LoadedSlice &RHS) {
14580 assert(LHS.Origin == RHS.Origin &&
"Different bases not implemented.");
14581 return LHS.getOffsetFromBase() < RHS.getOffsetFromBase();
14583 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
14586 const LoadedSlice *First =
nullptr;
14587 const LoadedSlice *Second =
nullptr;
14588 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
14591 Second = &LoadedSlices[CurrSlice];
14598 EVT LoadedType = First->getLoadedType();
14601 if (LoadedType != Second->getLoadedType())
14605 unsigned RequiredAlignment = 0;
14606 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
14612 if (RequiredAlignment > First->getAlignment())
14619 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
14620 --GlobalLSCost.Loads;
14637 const APInt &UsedBits,
bool ForCodeSize) {
14638 unsigned NumberOfSlices = LoadedSlices.
size();
14640 return NumberOfSlices > 1;
14643 if (NumberOfSlices != 2)
14651 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
14653 OrigCost.Loads = 1;
14654 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
14655 const LoadedSlice &LS = LoadedSlices[CurrSlice];
14657 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
14658 GlobalSlicingCost += SliceCost;
14662 OrigCost.addSliceGain(LS);
14667 return OrigCost > GlobalSlicingCost;
14676bool DAGCombiner::SliceUpLoad(
SDNode *
N) {
14682 !
LD->getValueType(0).isInteger())
14687 APInt UsedBits(
LD->getValueSizeInBits(0), 0);
14695 UI != UIEnd; ++UI) {
14697 if (UI.getUse().getResNo() != 0)
14701 unsigned Shift = 0;
14706 Shift =
User->getConstantOperandVal(1);
14719 unsigned Width =
User->getValueSizeInBits(0);
14724 LoadedSlice
LS(
User, LD, Shift, &DAG);
14725 APInt CurrentUsedBits =
LS.getUsedBits();
14728 if ((CurrentUsedBits & UsedBits) != 0)
14731 UsedBits |= CurrentUsedBits;
14753 LSIt = LoadedSlices.
begin(),
14754 LSItEnd = LoadedSlices.
end();
14755 LSIt != LSItEnd; ++LSIt) {
14756 SDValue SliceInst = LSIt->loadSlice();
14757 CombineTo(LSIt->Inst, SliceInst,
true);
14761 "It takes more than a zext to get to the loaded slice!!");
14768 AddToWorklist(Chain.
getNode());
14775static std::pair<unsigned, unsigned>
14777 std::pair<unsigned, unsigned> Result(0, 0);
14787 if (LD->getBasePtr() != Ptr)
return Result;
14798 uint64_t NotMask = ~cast<ConstantSDNode>(V->
getOperand(1))->getSExtValue();
14800 if (NotMaskLZ & 7)
return Result;
14802 if (NotMaskTZ & 7)
return Result;
14803 if (NotMaskLZ == 64)
return Result;
14814 switch (MaskedBytes) {
14818 default:
return Result;
14823 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
14834 if (ChainOp.getNode() == LD) {
14843 Result.first = MaskedBytes;
14844 Result.second = NotMaskTZ/8;
14855 unsigned NumBytes = MaskInfo.first;
14856 unsigned ByteShift = MaskInfo.second;
14862 ByteShift*8, (ByteShift+NumBytes)*8);
14869 if (!
DC->isTypeLegal(VT))
14886 StOffset = ByteShift;
14895 NewAlign =
MinAlign(NewAlign, StOffset);
14914 if (
ST->isVolatile())
14925 unsigned Opc =
Value.getOpcode();
14933 std::pair<unsigned, unsigned> MaskedLoad;
14935 if (MaskedLoad.first)
14937 Value.getOperand(1), ST,
this))
14942 if (MaskedLoad.first)
14944 Value.getOperand(0), ST,
this))
14956 if (
LD->getBasePtr() != Ptr ||
14957 LD->getPointerInfo().getAddrSpace() !=
14958 ST->getPointerInfo().getAddrSpace())
14964 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
14975 while (NewBW < BitWidth &&
14982 if (NewBW >= BitWidth)
14988 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
14990 std::min(BitWidth, ShAmt + NewBW));
14991 if ((Imm & Mask) == Imm) {
14995 uint64_t PtrOff = ShAmt / 8;
14999 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
15001 unsigned NewAlign =
MinAlign(
LD->getAlignment(), PtrOff);
15012 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
15013 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
15019 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
15021 AddToWorklist(NewPtr.
getNode());
15022 AddToWorklist(NewLD.
getNode());
15023 AddToWorklist(NewVal.
getNode());
15024 WorklistRemover DeadNodes(*
this);
15043 EVT VT =
LD->getMemoryVT();
15045 VT !=
ST->getMemoryVT() ||
15046 LD->isNonTemporal() ||
15047 ST->isNonTemporal() ||
15048 LD->getPointerInfo().getAddrSpace() != 0 ||
15049 ST->getPointerInfo().getAddrSpace() != 0)
15059 unsigned LDAlign =
LD->getAlignment();
15060 unsigned STAlign =
ST->getAlignment();
15063 if (LDAlign < ABIAlign || STAlign < ABIAlign)
15068 LD->getPointerInfo(), LDAlign);
15072 ST->getPointerInfo(), STAlign);
15074 AddToWorklist(NewLD.
getNode());
15075 AddToWorklist(NewST.
getNode());
15076 WorklistRemover DeadNodes(*
this);
15098bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
15109 if (
Use == MulNode)
15117 if (
Use->getOperand(0) == ConstNode)
15118 OtherOp =
Use->getOperand(1).getNode();
15120 OtherOp =
Use->getOperand(0).getNode();
15132 if (OtherOp == MulVar)
15160 unsigned NumStores) {
15163 SDLoc StoreDL(StoreNodes[0].MemNode);
15165 for (
unsigned i = 0; i < NumStores; ++i) {
15166 Visited.
insert(StoreNodes[i].MemNode);
15170 for (
unsigned i = 0; i < NumStores; ++i) {
15171 if (Visited.
insert(StoreNodes[i].MemNode->getChain().getNode()).second)
15172 Chains.
push_back(StoreNodes[i].MemNode->getChain());
15175 assert(Chains.
size() > 0 &&
"Chain should have generated a chain");
15179bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
15181 bool IsConstantSrc,
bool UseVector,
bool UseTrunc) {
15187 SDLoc DL(StoreNodes[0].MemNode);
15190 unsigned SizeInBits = NumStores * ElementSizeBits;
15195 unsigned Elts = NumStores * NumMemElts;
15203 if (IsConstantSrc) {
15205 for (
unsigned I = 0;
I != NumStores; ++
I) {
15206 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
15215 if (isa<ConstantFPSDNode>(Val)) {
15218 }
else if (
auto *
C = dyn_cast<ConstantSDNode>(Val))
15221 .zextOrTrunc(ElementSizeBits),
15231 DL, StoreTy, BuildVector);
15234 for (
unsigned i = 0; i < NumStores; ++i) {
15235 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
15269 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
15271 APInt StoreInt(SizeInBits, 0);
15276 for (
unsigned i = 0; i < NumStores; ++i) {
15277 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
15280 SDValue Val = St->getValue();
15282 StoreInt <<= ElementSizeBits;
15284 StoreInt |=
C->getAPIntValue()
15285 .zextOrTrunc(ElementSizeBits)
15286 .zextOrTrunc(SizeInBits);
15288 StoreInt |=
C->getValueAPF()
15290 .zextOrTrunc(ElementSizeBits)
15291 .zextOrTrunc(SizeInBits);
15301 StoredVal = DAG.
getConstant(StoreInt, DL, StoreTy);
15305 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
15314 EVT LegalizedStoredValTy =
15316 unsigned LegalizedStoreSize = LegalizedStoredValTy.
getSizeInBits();
15319 DAG.
getConstant(
C->getAPIntValue().zextOrTrunc(LegalizedStoreSize), DL,
15320 LegalizedStoredValTy);
15322 NewChain, DL, ExtendedStoreVal, FirstInChain->
getBasePtr(),
15329 for (
unsigned i = 0; i < NumStores; ++i)
15330 CombineTo(StoreNodes[i].MemNode, NewStore);
15332 AddToWorklist(NewChain.
getNode());
15336void DAGCombiner::getStoreMergeCandidates(
15346 if (!
BasePtr.getBase().getNode())
15350 if (
BasePtr.getBase().isUndef())
15353 bool IsConstantSrc = isa<ConstantSDNode>(Val) || isa<ConstantFPSDNode>(Val);
15356 bool IsLoadSrc = isa<LoadSDNode>(Val);
15361 auto *Ld = cast<LoadSDNode>(Val);
15363 LoadVT = Ld->getMemoryVT();
15365 if (MemVT != LoadVT)
15368 if (!Ld->hasNUsesOfValue(1, 0))
15371 if (Ld->isVolatile() || Ld->isIndexed())
15375 int64_t &
Offset) ->
bool {
15377 if (
Other->isVolatile() ||
Other->isIndexed())
15385 :
Other->getMemoryVT() != MemVT;
15390 if (
LoadSDNode *OtherLd = dyn_cast<LoadSDNode>(OtherBC)) {
15392 if (LoadVT != OtherLd->getMemoryVT())
15395 if (!OtherLd->hasNUsesOfValue(1, 0))
15398 if (OtherLd->isVolatile() || OtherLd->isIndexed())
15401 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
15408 if (IsConstantSrc) {
15411 if (!(isa<ConstantSDNode>(OtherBC) || isa<ConstantFPSDNode>(OtherBC)))
15414 if (IsExtractVecSrc) {
15416 if (
Other->isTruncatingStore())
15425 return (
BasePtr.equalBaseIndex(Ptr, DAG, Offset));
15446 unsigned NumNodesExplored = 0;
15447 if (
LoadSDNode *Ldn = dyn_cast<LoadSDNode>(RootNode)) {
15448 RootNode = Ldn->getChain().getNode();
15450 I !=
E && NumNodesExplored < 1024; ++
I, ++NumNodesExplored)
15451 if (
I.getOperandNo() == 0 && isa<LoadSDNode>(*
I))
15452 for (
auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2)
15453 if (I2.getOperandNo() == 0)
15454 if (
StoreSDNode *OtherST = dyn_cast<StoreSDNode>(*I2)) {
15457 if (CandidateMatch(OtherST, Ptr, PtrDiff))
15458 StoreNodes.
push_back(MemOpLink(OtherST, PtrDiff));
15462 I !=
E && NumNodesExplored < 1024; ++
I, ++NumNodesExplored)
15463 if (
I.getOperandNo() == 0)
15464 if (
StoreSDNode *OtherST = dyn_cast<StoreSDNode>(*
I)) {
15467 if (CandidateMatch(OtherST, Ptr, PtrDiff))
15468 StoreNodes.
push_back(MemOpLink(OtherST, PtrDiff));
15477bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
15493 while (!Worklist.
empty()) {
15495 if (!Visited.
insert(
N).second)
15504 unsigned int Max = 1024 + Visited.
size();
15506 for (
unsigned i = 0; i < NumStores; ++i) {
15507 SDNode *
N = StoreNodes[i].MemNode;
15519 for (
unsigned j = 1; j <
N->getNumOperands(); ++j)
15520 Worklist.
push_back(
N->getOperand(j).getNode());
15523 for (
unsigned i = 0; i < NumStores; ++i)
15530bool DAGCombiner::MergeConsecutiveStores(
StoreSDNode *St) {
15542 Attribute::NoImplicitFloat);
15554 bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
15555 bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
15556 isa<ConstantFPSDNode>(StoredVal);
15560 bool IsNonTemporalLoad =
15561 IsLoadSrc && cast<LoadSDNode>(StoredVal)->isNonTemporal();
15563 if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc)
15569 getStoreMergeCandidates(St, StoreNodes, RootNode);
15572 if (StoreNodes.
size() < 2)
15577 llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
15578 return LHS.OffsetFromBase < RHS.OffsetFromBase;
15590 while (StoreNodes.
size() > 1) {
15591 unsigned StartIdx = 0;
15592 while ((StartIdx + 1 < StoreNodes.
size()) &&
15593 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
15594 StoreNodes[StartIdx + 1].OffsetFromBase)
15598 if (StartIdx + 1 >= StoreNodes.
size())
15606 unsigned NumConsecutiveStores = 1;
15607 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
15610 for (
unsigned i = 1, e = StoreNodes.
size(); i < e; ++i) {
15611 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
15612 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
15614 NumConsecutiveStores = i + 1;
15617 if (NumConsecutiveStores < 2) {
15619 StoreNodes.
begin() + NumConsecutiveStores);
15628 if (IsConstantSrc) {
15629 while (NumConsecutiveStores >= 2) {
15632 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
15633 unsigned LastLegalType = 1;
15634 unsigned LastLegalVectorType = 1;
15635 bool LastIntegerTrunc =
false;
15636 bool NonZero =
false;
15637 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
15638 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
15639 StoreSDNode *
ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
15641 bool IsElementZero =
false;
15643 IsElementZero =
C->isNullValue();
15645 IsElementZero =
C->getConstantFPValue()->isNullValue();
15646 if (IsElementZero) {
15647 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
15648 FirstZeroAfterNonZero = i;
15650 NonZero |= !IsElementZero;
15653 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
15655 bool IsFast =
false;
15666 LastIntegerTrunc =
false;
15667 LastLegalType = i + 1;
15671 EVT LegalizedStoredValTy =
15679 LastIntegerTrunc =
true;
15680 LastLegalType = i + 1;
15691 unsigned Elts = (i + 1) * NumMemElts;
15696 Context, DL, Ty, *FirstInChain->
getMemOperand(), &IsFast) &&
15698 LastLegalVectorType = i + 1;
15702 bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors;
15703 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
15715 unsigned NumSkip = 1;
15717 (NumSkip < NumConsecutiveStores) &&
15718 (NumSkip < FirstZeroAfterNonZero) &&
15719 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
15723 NumConsecutiveStores -= NumSkip;
15728 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
15731 NumConsecutiveStores -= NumElem;
15735 RV |= MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
true,
15736 UseVector, LastIntegerTrunc);
15740 NumConsecutiveStores -= NumElem;
15747 if (IsExtractVecSrc) {
15749 while (NumConsecutiveStores >= 2) {
15752 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
15753 unsigned NumStoresToMerge = 1;
15754 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
15756 unsigned Elts = (i + 1) * NumMemElts;
15770 NumStoresToMerge = i + 1;
15775 if (NumStoresToMerge < 2) {
15782 unsigned NumSkip = 1;
15784 (NumSkip < NumConsecutiveStores) &&
15785 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
15789 NumConsecutiveStores -= NumSkip;
15794 if (!checkMergeStoreCandidatesForDependencies(
15795 StoreNodes, NumStoresToMerge, RootNode)) {
15797 StoreNodes.
begin() + NumStoresToMerge);
15798 NumConsecutiveStores -= NumStoresToMerge;
15802 RV |= MergeStoresOfConstantsOrVecElts(
15803 StoreNodes, MemVT, NumStoresToMerge,
false,
true,
false);
15806 StoreNodes.
begin() + NumStoresToMerge);
15807 NumConsecutiveStores -= NumStoresToMerge;
15823 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
15824 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
15830 int64_t LdOffset = 0;
15841 LoadNodes.
push_back(MemOpLink(Ld, LdOffset));
15844 while (NumConsecutiveStores >= 2 && LoadNodes.
size() >= 2) {
15847 unsigned RequiredAlignment;
15848 if (LoadNodes.
size() == 2 &&
15850 StoreNodes[0].MemNode->getAlignment() >= RequiredAlignment) {
15857 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
15858 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
15865 unsigned LastConsecutiveLoad = 1;
15868 unsigned LastLegalVectorType = 1;
15869 unsigned LastLegalIntegerType = 1;
15870 bool isDereferenceable =
true;
15871 bool DoIntegerTruncate =
false;
15872 StartAddress = LoadNodes[0].OffsetFromBase;
15874 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
15876 if (LoadNodes[i].MemNode->getChain() != FirstChain)
15879 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
15880 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
15882 LastConsecutiveLoad = i;
15884 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
15885 isDereferenceable =
false;
15888 unsigned Elts = (i + 1) * NumMemElts;
15895 bool IsFastSt, IsFastLd;
15904 LastLegalVectorType = i + 1;
15908 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
15918 LastLegalIntegerType = i + 1;
15919 DoIntegerTruncate =
false;
15938 LastLegalIntegerType = i + 1;
15939 DoIntegerTruncate =
true;
15947 LastLegalVectorType > LastLegalIntegerType && !NoVectors;
15948 unsigned LastLegalType =
15949 std::max(LastLegalVectorType, LastLegalIntegerType);
15954 std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
15955 NumElem = std::min(LastLegalType, NumElem);
15965 unsigned NumSkip = 1;
15966 while ((NumSkip < LoadNodes.
size()) &&
15967 (LoadNodes[NumSkip].MemNode->getAlignment() <= FirstLoadAlign) &&
15968 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
15972 NumConsecutiveStores -= NumSkip;
15977 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
15981 NumConsecutiveStores -= NumElem;
15990 unsigned Elts = NumElem * NumMemElts;
15993 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
15997 SDLoc LoadDL(LoadNodes[0].MemNode);
15998 SDLoc StoreDL(StoreNodes[0].MemNode);
16003 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
16004 AddToWorklist(NewStoreChain.
getNode());
16009 if (IsNonTemporalLoad)
16017 if (UseVectorTy || !DoIntegerTruncate) {
16021 FirstLoadAlign, LdMMOFlags);
16023 NewStoreChain, StoreDL, NewLoad, FirstInChain->
getBasePtr(),
16031 FirstLoadAlign, LdMMOFlags);
16032 NewStore = DAG.
getTruncStore(NewStoreChain, StoreDL, NewLoad,
16040 for (
unsigned i = 0; i < NumElem; ++i) {
16041 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
16048 for (
unsigned i = 0; i < NumElem; ++i) {
16049 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
16050 CombineTo(StoreNodes[i].MemNode, NewStore);
16052 recursivelyDeleteUnusedNodes(Val.
getNode());
16058 NumConsecutiveStores -= NumElem;
16069 if (
ST->isTruncatingStore()) {
16071 ST->getBasePtr(),
ST->getMemoryVT(),
16072 ST->getMemOperand());
16074 ReplStore = DAG.
getStore(BetterChain, SL,
ST->getValue(),
ST->getBasePtr(),
16075 ST->getMemOperand());
16083 AddToWorklist(Token.
getNode());
16086 return CombineTo(ST, Token,
false);
16120 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
16122 return DAG.
getStore(Chain, DL, Tmp, Ptr,
ST->getMemOperand());
16128 !
ST->isVolatile()) ||
16133 return DAG.
getStore(Chain, DL, Tmp,
16134 Ptr,
ST->getMemOperand());
16137 if (!
ST->isVolatile() &&
16148 unsigned Alignment =
ST->getAlignment();
16153 ST->getAlignment(), MMOFlags, AAInfo);
16156 Alignment =
MinAlign(Alignment, 4U);
16158 ST->getPointerInfo().getWithOffset(4),
16159 Alignment, MMOFlags, AAInfo);
16177 ST->isUnindexed()) {
16178 EVT SVT =
Value.getOperand(0).getValueType();
16184 if (((!LegalOperations && !
ST->isVolatile()) ||
16187 DAG, *
ST->getMemOperand())) {
16189 ST->getPointerInfo(),
ST->getAlignment(),
16190 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
16195 if (
Value.isUndef() &&
ST->isUnindexed())
16201 if (Align >
ST->getAlignment() &&
ST->getSrcValueOffset() % Align == 0) {
16204 ST->getMemoryVT(), Align,
16205 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
16215 if (
SDValue NewST = TransformFPLoadStorePair(
N))
16219 if (
SDValue Store = MatchStoreCombine(ST))
16222 if (
ST->isUnindexed()) {
16225 if (findBetterNeighborChains(ST)) {
16230 Chain =
ST->getChain();
16234 if (
ST->isTruncatingStore() &&
ST->isUnindexed() &&
16235 Value.getValueType().isInteger() &&
16236 (!isa<ConstantSDNode>(
Value) ||
16237 !cast<ConstantSDNode>(
Value)->isOpaque())) {
16238 APInt TruncDemandedBits =
16240 ST->getMemoryVT().getScalarSizeInBits());
16246 AddToWorklist(
Value.getNode());
16249 ST->getMemOperand());
16268 ST->isUnindexed() && !
ST->isVolatile() &&
16277 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
16278 if (
ST->isUnindexed() && !
ST->isVolatile() && ST1->isUnindexed() &&
16279 !ST1->isVolatile()) {
16280 if (ST1->getBasePtr() == Ptr && ST1->getValue() ==
Value &&
16281 ST->getMemoryVT() == ST1->getMemoryVT()) {
16288 !ST1->getBasePtr().isUndef()) {
16291 unsigned STBitSize =
ST->getMemoryVT().getSizeInBits();
16292 unsigned ChainBitSize = ST1->getMemoryVT().getSizeInBits();
16297 if (STBase.
contains(DAG, STBitSize, ChainBase, ChainBitSize)) {
16298 CombineTo(ST1, ST1->getChain());
16307 if (ChainBase.
contains(DAG, ChainBitSize, STBase, STBitSize,
16309 SDValue ChainValue = ST1->getValue();
16310 if (
auto *C1 = dyn_cast<ConstantSDNode>(ChainValue)) {
16311 if (
auto *
C = dyn_cast<ConstantSDNode>(
Value)) {
16312 APInt Val = C1->getAPIntValue();
16313 APInt InsertVal =
C->getAPIntValue().zextOrTrunc(STBitSize);
16319 C1->isTargetOpcode(), C1->isOpaque());
16321 ST1, ST1->getChain(), NewSDVal, ST1->getOperand(2),
16322 ST1->getOperand(3));
16323 return CombineTo(ST,
SDValue(NewST1, 0));
16337 ST->getMemoryVT())) {
16339 Ptr,
ST->getMemoryVT(),
ST->getMemOperand());
16350 bool Changed = MergeConsecutiveStores(ST);
16351 if (!Changed)
break;
16360 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
16368 if (isa<ConstantFPSDNode>(
ST->getValue())) {
16369 if (
SDValue NewSt = replaceStoreOfFPConstant(ST))
16376 return ReduceLoadOpStoreWidth(
N);
16380 const auto *LifetimeEnd = cast<LifetimeSDNode>(
N);
16381 if (!LifetimeEnd->hasOffset())
16385 LifetimeEnd->getOffset(),
false);
16389 while (!Chains.
empty()) {
16408 if (
ST->isVolatile() ||
ST->isIndexed())
16413 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
16414 ST->getMemoryVT().getStoreSizeInBits())) {
16416 dbgs() <<
"\nwithin LIFETIME_END of : ";
16417 LifetimeEndBase.dump();
dbgs() <<
"\n");
16418 CombineTo(ST,
ST->getChain());
16489 !
Lo.getOperand(0).getValueType().isScalarInteger() ||
16490 Lo.getOperand(0).getValueSizeInBits() > HalfValBitSize ||
16492 !
Hi.getOperand(0).getValueType().isScalarInteger() ||
16493 Hi.getOperand(0).getValueSizeInBits() > HalfValBitSize)
16499 ?
Lo.getOperand(0).getValueType()
16500 :
Lo.getValueType();
16502 ?
Hi.getOperand(0).getValueType()
16503 :
Hi.getValueType();
16508 unsigned Alignment =
ST->getAlignment();
16521 ST->getAlignment(), MMOFlags, AAInfo);
16528 ST->getPointerInfo().getWithOffset(HalfValBitSize / 8),
16529 Alignment / 2, MMOFlags, AAInfo);
16538SDValue DAGCombiner::combineInsertEltToShuffle(
SDNode *
N,
unsigned InsIndex) {
16539 SDValue InsertVal =
N->getOperand(1);
16545 SDValue DestVec =
N->getOperand(0);
16550 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
16558 for (
unsigned i = 0; i != NumMaskVals; ++i) {
16559 if (i / NumSrcElts == InsIndex)
16560 Mask[i] = (i % NumSrcElts) + NumMaskVals;
16575 ConcatOps[0] = SubVec;
16581 AddToWorklist(PaddedSubV.
getNode());
16582 AddToWorklist(DestVecBC.
getNode());
16583 AddToWorklist(Shuf.
getNode());
16606 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
16618 unsigned Elt = IndexC->getZExtValue();
16619 if (
SDValue Shuf = combineInsertEltToShuffle(
N, Elt))
16630 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
16632 if (Elt < OtherElt) {
16636 AddToWorklist(NewOp.
getNode());
16655 }
else if (InVec.
isUndef()) {
16660 assert(Ops.
size() == NumElts &&
"Unexpected vector size");
16663 if (Elt < Ops.
size()) {
16666 EVT OpVT = Ops[0].getValueType();
16674SDValue DAGCombiner::scalarizeExtractedVectorLoad(
SDNode *EVE,
EVT InVecVT,
16700 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
16701 int Elt = ConstEltNo->getZExtValue();
16724 if (ResultVT.
bitsGT(VecEltVT)) {
16732 OriginalLoad->
getChain(), NewPtr, MPI, VecEltVT,
16735 Chain =
Load.getValue(1);
16740 Chain =
Load.getValue(1);
16741 if (ResultVT.
bitsLT(VecEltVT))
16746 WorklistRemover DeadNodes(*
this);
16752 AddToWorklist(
Load.getNode());
16753 AddUsersToWorklist(
Load.getNode());
16755 AddToWorklist(EVE);
16763 bool LegalOperations) {
16767 auto *IndexC = dyn_cast<ConstantSDNode>(Index);
16798 EVT ScalarVT =
N->getValueType(0);
16828 auto *IndexC = dyn_cast<ConstantSDNode>(Index);
16830 if (IndexC && IndexC->getAPIntValue().uge(NumElts))
16841 if (ScalarVT == InEltVT)
16855 unsigned ExtractIndex = IndexC->getZExtValue();
16857 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
16868 "Extract element and scalar to vector can't change element type "
16869 "from FP to integer.");
16870 unsigned XBitWidth =
X.getValueSizeInBits();
16872 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
16877 if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
16878 assert(XBitWidth % VecEltBitWidth == 0 &&
16879 "Scalar bitwidth must be a multiple of vector element bitwidth");
16895 auto *Shuf = cast<ShuffleVectorSDNode>(VecOp);
16897 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue());
16905 if (OrigElt < (
int)NumElts) {
16909 OrigElt -= NumElts;
16925 if (!LegalOperations ||
16938 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16939 Use->getOperand(0) == VecOp &&
16940 isa<ConstantSDNode>(Use->getOperand(1));
16944 auto *CstElt = cast<ConstantSDNode>(
Use->getOperand(1));
16945 if (CstElt->getAPIntValue().ult(NumElts))
16946 DemandedElts.
setBit(CstElt->getZExtValue());
16960 bool BCNumEltsChanged =
false;
16975 BCNumEltsChanged =
true;
16981 if (!LegalOperations && !IndexC && VecOp.
hasOneUse() &&
16984 auto *
VecLoad = dyn_cast<LoadSDNode>(VecOp);
16985 if (VecLoad && !
VecLoad->isVolatile())
16986 return scalarizeExtractedVectorLoad(
N, VecVT, Index, VecLoad);
16991 if (!LegalOperations || !IndexC)
16997 int Elt = IndexC->getZExtValue();
17000 LN0 = cast<LoadSDNode>(VecOp);
17008 LN0 = cast<LoadSDNode>(VecOp.
getOperand(0));
17010 if (
auto *Shuf = dyn_cast<ShuffleVectorSDNode>(VecOp)) {
17021 if (BCNumEltsChanged)
17025 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt);
17036 LN0 = cast<LoadSDNode>(VecOp);
17037 Elt = (
Idx < (int)NumElts) ?
Idx :
Idx - (int)NumElts;
17051 return scalarizeExtractedVectorLoad(
N, VecVT, Index, LN0);
17065 unsigned NumInScalars =
N->getNumOperands();
17067 EVT VT =
N->getValueType(0);
17075 bool AllAnyExt =
true;
17077 for (
unsigned i = 0; i != NumInScalars; ++i) {
17080 if (
In.isUndef())
continue;
17086 if (!ZeroExt && !AnyExt) {
17092 EVT InTy =
In.getOperand(0).getValueType();
17098 else if (InTy != SourceType) {
17105 AllAnyExt &= AnyExt;
17112 bool ValidTypes = SourceType !=
MVT::Other &&
17123 assert(ElemRatio > 1 &&
"Invalid element size ratio");
17131 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
17135 Cast.
isUndef()) &&
"Invalid cast opcode");
17141 unsigned Index = isLE ? (i * ElemRatio) :
17142 (i * ElemRatio + (ElemRatio - 1));
17144 assert(Index < Ops.
size() &&
"Invalid index");
17151 "Invalid vector size");
17170 unsigned LeftIdx,
bool DidSplitVec) {
17174 EVT VT =
N->getValueType(0);
17179 unsigned ShuffleNumElems = NumElems;
17187 if (InVT1 != VT || InVT2 != VT) {
17192 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
17194 ConcatOps[0] = VecIn1;
17195 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
17210 Vec2Offset = NumElems;
17216 if (LegalOperations &&
17223 if (InVT1 != InVT2) {
17227 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
17229 ShuffleNumElems = NumElems * 2;
17239 ConcatOps[0] = VecIn2;
17256 for (
unsigned i = 0; i != NumElems; ++i) {
17257 if (VectorMask[i] <= 0)
17260 unsigned ExtIndex =
N->getOperand(i).getConstantOperandVal(1);
17261 if (VectorMask[i] == (
int)LeftIdx) {
17262 Mask[i] = ExtIndex;
17263 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
17264 Mask[i] = Vec2Offset + ExtIndex;
17277 if (ShuffleNumElems > NumElems)
17290 for (
int i = 0; i != NumBVOps; ++i) {
17322 if (DestSize % SrcSize != 0 ||
17328 int ZextRatio = DestSize / SrcSize;
17329 int NumMaskElts = NumBVOps * ZextRatio;
17331 for (
int i = 0; i != NumMaskElts; ++i) {
17332 if (i / ZextRatio == ZextElt) {
17337 if (i % ZextRatio == 0)
17340 ShufMask[i] = NumMaskElts;
17366 EVT VT =
N->getValueType(0);
17379 bool UsesZeroVector =
false;
17380 unsigned NumElems =
N->getNumOperands();
17392 for (
unsigned i = 0; i != NumElems; ++i) {
17402 UsesZeroVector =
true;
17410 !isa<ConstantSDNode>(
Op.getOperand(1)))
17412 SDValue ExtractedFromVec =
Op.getOperand(0);
17414 const APInt &ExtractIdx =
Op.getConstantOperandAPInt(1);
17426 unsigned Idx = std::distance(
17427 VecIn.
begin(), std::find(VecIn.
begin(), VecIn.
end(), ExtractedFromVec));
17431 VectorMask[i] =
Idx;
17435 if (VecIn.
size() < 2)
17442 bool DidSplitVec =
false;
17443 if (VecIn.
size() == 2) {
17444 unsigned MaxIndex = 0;
17445 unsigned NearestPow2 = 0;
17451 for (
unsigned i = 0; i < NumElems; i++) {
17452 if (VectorMask[i] <= 0)
17454 unsigned Index =
N->getOperand(i).getConstantOperandVal(1);
17455 IndexVec[i] =
Index;
17456 MaxIndex = std::max(MaxIndex, Index);
17460 if (InVT.
isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
17461 NumElems * 2 < NearestPow2) {
17462 unsigned SplitSize = NearestPow2 / 2;
17473 DidSplitVec =
true;
17475 for (
unsigned i = 0; i < NumElems; i++) {
17476 if (VectorMask[i] <= 0)
17478 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
17504 for (
unsigned In = 0, Len = (VecIn.
size() / 2);
In < Len; ++
In) {
17505 unsigned LeftIdx = 2 *
In + 1;
17506 SDValue VecLeft = VecIn[LeftIdx];
17508 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
17510 if (
SDValue Shuffle = createBuildVecShuffle(DL,
N, VectorMask, VecLeft,
17511 VecRight, LeftIdx, DidSplitVec))
17519 if (UsesZeroVector)
17524 if (Shuffles.
size() == 1)
17525 return Shuffles[0];
17528 for (
int &Vec : VectorMask)
17530 Vec = Shuffles.
size() - 1;
17532 Vec = (Vec - 1) / 2;
17546 if (Shuffles.
size() % 2)
17549 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
17551 Shuffles[CurSize] = DAG.
getUNDEF(VT);
17554 for (
unsigned In = 0, Len = CurSize / 2;
In < Len; ++
In) {
17558 for (
unsigned i = 0; i != NumElems; ++i) {
17559 if (VectorMask[i] ==
Left) {
17561 VectorMask[i] =
In;
17562 }
else if (VectorMask[i] ==
Right) {
17563 Mask[i] = i + NumElems;
17564 VectorMask[i] =
In;
17572 return Shuffles[0];
17580 if (LegalOperations)
17583 EVT VT =
N->getValueType(0);
17585 bool FoundZeroExtend =
false;
17587 auto checkElem = [&](
SDValue Op) -> int64_t {
17588 unsigned Opc =
Op.getOpcode();
17593 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op.getOperand(0).getOperand(1)))
17594 return C->getZExtValue();
17600 int64_t
Offset = checkElem(Op0);
17604 unsigned NumElems =
N->getNumOperands();
17606 EVT InSVT =
In.getValueType().getScalarType();
17614 for (
unsigned i = 1; i != NumElems; ++i) {
17615 if ((Offset + i) != checkElem(
N->getOperand(i)))
17627 EVT VT =
N->getValueType(0);
17641 if (!LegalOperations) {
17644 EVT SrcVT = Splat.getValueType();
17660 if (!LegalTypes && (
N->getNumOperands() > 1)) {
17662 auto checkElem = [&](
SDValue Op) -> uint64_t {
17665 if (
auto CNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
17666 return CNode->getZExtValue();
17670 int Offset = checkElem(Op0);
17671 for (
unsigned i = 0; i <
N->getNumOperands(); ++i) {
17672 if (Offset + i != checkElem(
N->getOperand(i))) {
17678 if ((Offset == 0) &&
17681 if ((Offset != -1) &&
17682 ((Offset %
N->getValueType(0).getVectorNumElements()) ==
17688 if (
SDValue V = convertBuildVecZextToZext(
N))
17691 if (
SDValue V = reduceBuildVecExtToExtBuildVec(
N))
17694 if (
SDValue V = reduceBuildVecToShuffle(
N))
17702 EVT OpVT =
N->getOperand(0).getValueType();
17709 EVT VT =
N->getValueType(0);
17716 bool AnyInteger =
false;
17717 bool AnyFP =
false;
17718 for (
const SDValue &Op :
N->ops()) {
17720 !Op.getOperand(0).getValueType().isVector())
17729 EVT LastOpVT = Ops.
back().getValueType();
17746 if (Op.getValueType() == SVT)
17766 EVT VT =
N->getValueType(0);
17767 EVT OpVT =
N->getOperand(0).getValueType();
17778 if (Op.isUndef()) {
17779 Mask.append((
unsigned)NumOpElts, -1);
17787 SDValue ExtVec = Op.getOperand(0);
17796 Mask.append((
unsigned)NumOpElts, -1);
17800 if (!isa<ConstantSDNode>(Op.getOperand(1)))
17802 int ExtIdx = Op.getConstantOperandVal(1);
17811 if (0 == (NumExtElts % NumElts))
17812 ExtIdx /= (NumExtElts / NumElts);
17813 else if (0 == (NumElts % NumExtElts))
17814 ExtIdx *= (NumElts / NumExtElts);
17819 if (SV0.
isUndef() || SV0 == ExtVec) {
17821 for (
int i = 0; i != NumOpElts; ++i)
17822 Mask.push_back(i + ExtIdx);
17823 }
else if (SV1.
isUndef() || SV1 == ExtVec) {
17825 for (
int i = 0; i != NumOpElts; ++i)
17826 Mask.push_back(i + ExtIdx + NumElts);
17841 if (
N->getNumOperands() == 1)
17842 return N->getOperand(0);
17845 EVT VT =
N->getValueType(0);
17850 if (std::all_of(std::next(
N->op_begin()),
N->op_end(), [](
const SDValue &Op) {
17851 return Op.isUndef();
17854 assert(
In.getValueType().isVector() &&
"Must concat vectors");
17862 EVT SVT =
Scalar.getValueType().getVectorElementType();
17863 if (SVT ==
Scalar.getOperand(0).getValueType())
17868 if (!
Scalar.getValueType().isVector()) {
17887 if (VNTNumElms < 2)
17903 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
17914 bool FoundMinVT =
false;
17915 for (
const SDValue &Op :
N->ops())
17917 EVT OpSVT =
Op.getOperand(0).getValueType();
17918 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
17921 assert(FoundMinVT &&
"Concat vector type mismatch");
17924 for (
const SDValue &Op :
N->ops()) {
17925 EVT OpVT =
Op.getValueType();
17934 Opnds.
append(
Op->op_begin(),
Op->op_begin() + NumElts);
17936 for (
unsigned i = 0; i != NumElts; ++i)
17944 "Concat vector type mismatch");
17962 unsigned PartNumElem =
N->getOperand(0).getValueType().getVectorNumElements();
17964 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
17975 if (SingleSource.
getNode()) {
17976 if (
Op.getOperand(0) != SingleSource)
17979 SingleSource =
Op.getOperand(0);
17988 auto *CS = dyn_cast<ConstantSDNode>(
Op.getOperand(1));
17994 unsigned IdentityIndex = i * PartNumElem;
17995 if (CS->getAPIntValue() != IdentityIndex)
18000 return SingleSource;
18009 unsigned BinOpcode = BinOp.
getOpcode();
18021 V.getOperand(1).getValueType() == VT && V.getOperand(2) == Index) {
18022 return V.getOperand(1);
18024 auto *IndexC = dyn_cast<ConstantSDNode>(Index);
18026 V.getOperand(0).getValueType() == VT &&
18029 return V.getOperand(SubIdx);
18033 SDValue Sub0 = GetSubVector(Bop0);
18034 SDValue Sub1 = GetSubVector(Bop1);
18045 return DAG.
getNode(BinOpcode,
SDLoc(Extract), VT, Sub0, Sub1,
18059 auto *ExtractIndexC = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
18060 if (!ExtractIndexC)
18077 unsigned ExtractIndex = ExtractIndexC->getZExtValue();
18079 "Extract index is not a multiple of the vector length.");
18084 if (WideWidth % NarrowWidth != 0)
18089 unsigned NarrowingRatio = WideWidth / NarrowWidth;
18091 if (WideNumElts % NarrowingRatio != 0)
18096 WideNumElts / NarrowingRatio);
18123 if (NarrowingRatio != 2)
18138 return V.getOperand(ConcatOpNum);
18144 if (SubVecL || SubVecR) {
18175 auto *Ld = dyn_cast<LoadSDNode>(Extract->
getOperand(0));
18176 auto *ExtIdx = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
18203 EVT NVT =
N->getValueType(0);
18235 if ((SrcNumElts % DestNumElts) == 0) {
18236 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
18241 unsigned IndexValScaled =
N->getConstantOperandVal(1) * SrcDestRatio;
18260 unsigned Idx =
N->getConstantOperandVal(1);
18263 "IDX in concat is not a multiple of the result vector length.");
18271 if (
auto *IdxC = dyn_cast<ConstantSDNode>(Index)) {
18276 if (ExtractSize % EltSize == 0) {
18277 unsigned NumElems = ExtractSize / EltSize;
18279 EVT ExtractVT = NumElems == 1 ? EltVT
18286 unsigned IdxVal = IdxC->getZExtValue();
18290 if (NumElems == 1) {
18299 ExtractVT,
SDLoc(
N), V->
ops().slice(IdxVal, NumElems));
18310 if (!NVT.
bitsEq(SmallVT))
18314 auto *ExtIdx = dyn_cast<ConstantSDNode>(Index);
18315 auto *InsIdx = dyn_cast<ConstantSDNode>(V.
getOperand(2));
18316 if (InsIdx && ExtIdx) {
18358 unsigned HalfNumElts = NumElts / 2;
18361 for (
unsigned i = 0; i != NumElts; ++i) {
18364 int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (
int)HalfNumElts;
18365 if (i < HalfNumElts)
18368 Mask1[i - HalfNumElts] = M;
18391 EVT VT =
N->getValueType(0);
18402 unsigned NumConcats = NumElts / NumElemsPerConcat;
18404 auto IsUndefMaskElt = [](
int i) {
return i == -1; };
18409 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
18410 llvm::all_of(Mask.slice(NumElemsPerConcat, NumElemsPerConcat),
18414 Mask.slice(0, NumElemsPerConcat));
18421 for (
unsigned I = 0;
I != NumConcats; ++
I) {
18422 unsigned Begin =
I * NumElemsPerConcat;
18423 ArrayRef<int> SubMask = Mask.slice(Begin, NumElemsPerConcat);
18432 for (
int i = 0; i != (int)NumElemsPerConcat; ++i) {
18433 if (IsUndefMaskElt(SubMask[i]))
18435 if ((SubMask[i] % (
int)NumElemsPerConcat) != i)
18437 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
18438 if (0 <= OpIdx && EltOpIdx != OpIdx)
18442 assert(0 <= OpIdx &&
"Unknown concat_vectors op");
18496 bool IsSplat =
false;
18497 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
18498 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
18500 if (
SDValue Splat0 = BV0->getSplatValue())
18501 IsSplat = (Splat0 == BV1->getSplatValue());
18505 for (
int M : SVN->
getMask()) {
18508 int Idx = M < (int)NumElts ? M : M - NumElts;
18509 SDValue &S = (M < (int)NumElts ? N0 : N1);
18511 Op = S.getOperand(
Idx);
18525 if (!Op.isUndef() && !isa<ConstantSDNode>(Op) && !isa<ConstantFPSDNode>(Op))
18526 if (!IsSplat && !DuplicateOps.
insert(Op).second)
18537 SVT = (SVT.
bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
18553 bool LegalOperations) {
18567 auto isAnyExtend = [&Mask, &NumElts](
unsigned Scale) {
18568 for (
unsigned i = 0; i != NumElts; ++i) {
18571 if ((i % Scale) == 0 && Mask[i] == (int)(i / Scale))
18580 for (
unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
18582 if (NumElts % Scale != 0)
18584 if (!isAnyExtend(Scale))
18592 if (!LegalOperations ||
18596 SDLoc(SVN), OutVT, N0));
18631 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
18633 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
18638 auto isTruncate = [&Mask, &NumElts](
unsigned Scale) {
18639 for (
unsigned i = 0; i != NumElts; ++i) {
18642 if ((i * Scale) < NumElts && Mask[i] == (int)(i * Scale))
18652 if (EltSizeInBits != ExtSrcSizeInBits)
18657 if (isTruncate(ExtScale))
18672 auto *Splat = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
18673 if (!Splat || !Splat->isSplat())
18678 assert(ShufMask.
size() == SplatMask.
size() &&
"Mask length mismatch");
18696 auto CanSimplifyToExistingSplat = [](
ArrayRef<int> UserMask,
18698 for (
unsigned i = 0, e = UserMask.
size(); i != e; ++i)
18699 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
18700 SplatMask[UserMask[i]] != -1)
18704 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
18710 for (
int Idx : ShufMask)
18714 Splat->getOperand(0), Splat->getOperand(1),
18723 int MaskSize = Mask.size();
18724 int EltFromOp0 = -1;
18729 for (
int i = 0; i != MaskSize; ++i) {
18730 if (Mask[i] >= 0 && Mask[i] < MaskSize) {
18732 if (EltFromOp0 != -1)
18735 }
else if (Mask[i] != i + MaskSize) {
18755 if (ShufOp0Index == -1) {
18759 if (ShufOp0Index == -1)
18763 Mask = CommutedMask;
18771 assert(Mask[ShufOp0Index] >= 0 && Mask[ShufOp0Index] < (
int)Mask.size() &&
18772 "Shuffle mask value must be from operand 0");
18776 auto *InsIndexC = dyn_cast<ConstantSDNode>(Op0.
getOperand(2));
18777 if (!InsIndexC || InsIndexC->getSExtValue() != Mask[ShufOp0Index])
18802 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
18808 for (
int i = 0, e = (
int)Mask.size(); i != e; ++i) {
18812 assert(Mask[i] >= 0 && Mask[i] < e &&
"Unexpected shuffle mask value");
18816 if (Mask0[Mask[i]] != Mask0[i])
18825 EVT VT =
N->getValueType(0);
18842 for (
unsigned i = 0; i != NumElts; ++i) {
18844 if (
Idx >= (
int)NumElts)
Idx -= NumElts;
18856 bool Changed =
false;
18858 for (
unsigned i = 0; i != NumElts; ++i) {
18860 if (
Idx >= (
int)NumElts) {
18911 "BUILD_VECTOR has wrong number of operands");
18913 bool AllSame =
true;
18914 for (
unsigned i = 0; i != NumElts; ++i) {
18923 for (
unsigned i = 0; i != NumElts; ++i) {
18990 for (
int s = 0; s != Scale; ++s)
18991 NewMask.
push_back(M < 0 ? -1 : Scale * M + s);
19002 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
19014 ScaleShuffleMask(InnerSVN->
getMask(), InnerScale);
19016 ScaleShuffleMask(SVN->
getMask(), OuterScale);
19020 for (
int M : OuterMask)
19021 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
19053 "Shuffle types don't match");
19057 bool HasSameOp0 = N0 == SV0;
19058 bool IsSV1Undef = SV1.
isUndef();
19059 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
19083 "Shuffle types don't match");
19089 for (
unsigned i = 0; i != NumElts; ++i) {
19098 if (
Idx < (
int)NumElts) {
19108 CurrentVec = (
Idx < (int) NumElts) ? OtherSV->
getOperand(0)
19117 Mask.push_back(-1);
19124 if (!SV0.
getNode() || SV0 == CurrentVec) {
19133 if (SV1.
getNode() && SV1 != CurrentVec)
19139 Mask.push_back(
Idx + NumElts);
19143 bool isUndefMask =
true;
19144 for (
unsigned i = 0; i != NumElts && isUndefMask; ++i)
19145 isUndefMask &= Mask[i] < 0;
19182 EVT VT =
N->getValueType(0);
19232 EVT VT =
N->getValueType(0);
19296 if (!isa<ConstantSDNode>(N2))
19299 uint64_t InsIdx = cast<ConstantSDNode>(N2)->getZExtValue();
19310 if ((N0.
isUndef() || N0SrcSVT == N1SrcSVT) &&
19322 NewIdx = DAG.
getConstant(InsIdx * Scale, DL, IdxVT);
19323 }
else if ((N1SrcSVT.
getSizeInBits() % EltSizeInBits) == 0) {
19325 if ((NumElts % Scale) == 0 && (InsIdx % Scale) == 0) {
19327 NewIdx = DAG.
getConstant(InsIdx / Scale, DL, IdxVT);
19346 if (InsIdx < OtherIdx) {
19350 AddToWorklist(NewOp.
getNode());
19363 Ops[cast<ConstantSDNode>(N2)->getZExtValue() / Factor] = N1;
19403 unsigned Opcode =
N->getOpcode();
19424 return DAG.
getNode(NewOpcode,
SDLoc(
N),
N->getValueType(0), N0);
19437 EVT VT =
N->getValueType(0);
19444 if (LegalOperations)
19456 auto BuildClearMask = [&](
int Split) {
19457 int NumSubElts = NumElts * Split;
19461 for (
int i = 0; i != NumSubElts; ++i) {
19462 int EltIdx = i / Split;
19463 int SubIdx = i % Split;
19471 if (isa<ConstantSDNode>(Elt))
19472 Bits = cast<ConstantSDNode>(Elt)->getAPIntValue();
19473 else if (isa<ConstantFPSDNode>(Elt))
19474 Bits = cast<ConstantFPSDNode>(Elt)->getValueAPF().bitcastToAPInt();
19480 Bits.lshrInPlace((Split - SubIdx - 1) * NumSubBits);
19482 Bits.lshrInPlace(SubIdx * NumSubBits);
19488 if (
Bits.isAllOnesValue())
19490 else if (Bits == 0)
19513 for (
int Split = 1; Split <= MaxSplit; ++Split)
19515 if (
SDValue S = BuildClearMask(Split))
19526 unsigned Opcode =
N->getOpcode();
19527 EVT VT =
N->getValueType(0);
19534 int Index0, Index1;
19537 if (!Src0 || !Src1 || Index0 != Index1 ||
19559 Ops[Index0] = ScalarBO;
19570 assert(
N->getValueType(0).isVector() &&
19571 "SimplifyVBinOp only works on vectors!");
19576 EVT VT =
N->getValueType(0);
19577 unsigned Opcode =
N->getOpcode();
19594 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(LHS);
19595 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(RHS);
19596 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
19618 EVT NarrowVT =
X.getValueType();
19619 if (NarrowVT ==
Y.getValueType() &&
19641 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
19655 AddToWorklist(
SETCC.getNode());
19672bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
19685 CC = cast<CondCodeSDNode>(TheSelect->
getOperand(4))->get();
19692 CC = cast<CondCodeSDNode>(Cmp.getOperand(2))->get();
19693 CmpLHS = Cmp.getOperand(0);
19697 if (Zero && Zero->
isZero() &&
19701 CombineTo(TheSelect, Sqrt);
19766 Visited.
insert(TheSelect);
19845 CombineTo(TheSelect, Load);
19891 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
19892 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
19893 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
19896 AddToWorklist(Shift.
getNode());
19898 if (XType.
bitsGT(AType)) {
19900 AddToWorklist(Shift.
getNode());
19904 Shift = DAG.
getNOT(DL, Shift, AType);
19911 AddToWorklist(Shift.
getNode());
19913 if (XType.
bitsGT(AType)) {
19915 AddToWorklist(Shift.
getNode());
19919 Shift = DAG.
getNOT(DL, Shift, AType);
19928SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
19936 auto *TV = dyn_cast<ConstantFPSDNode>(N2);
19937 auto *FV = dyn_cast<ConstantFPSDNode>(N3);
19944 TLI.
isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) ||
19945 TLI.
isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize))
19950 if (!TV->hasOneUse() && !FV->hasOneUse())
19954 const_cast<ConstantFP*
>(TV->getConstantFPValue()) };
19962 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
19971 AddToWorklist(Cond.getNode());
19973 AddToWorklist(CstOffset.
getNode());
19975 AddToWorklist(CPIdx.
getNode());
19985 bool NotExtCompare) {
19987 if (N2 == N3)
return N2;
19992 auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode());
19993 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
19994 auto *N3C = dyn_cast<ConstantSDNode>(N3.
getNode());
19998 AddToWorklist(SCC.
getNode());
19999 if (
auto *SCCC = dyn_cast<ConstantSDNode>(SCC)) {
20002 return !(SCCC->isNullValue()) ? N2 : N3;
20007 convertSelectOfFPConstantsToLoadOffset(DL, N0, N1, N2, N3, CC))
20010 if (
SDValue V = foldSelectCCToShiftAnd(DL, N0, N1, N2, N3, CC))
20022 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
20023 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
20025 const APInt &AndMask = ConstAndRHS->getAPIntValue();
20043 bool Fold = N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2();
20044 bool Swap = N3C &&
isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2();
20046 if ((Fold || Swap) &&
20058 if (NotExtCompare && N2C->isOne())
20064 SCC = DAG.
getSetCC(DL, CmpResVT, N0, N1, CC);
20074 AddToWorklist(SCC.
getNode());
20075 AddToWorklist(Temp.
getNode());
20102 if (
auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(ValueOnZero)) {
20128 bool foldBooleans) {
20130 DagCombineInfo(DAG, Level,
false,
this);
20131 return TLI.
SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL);
20162 if (
C->isNullValue())
20217 EVT VT =
Op.getValueType();
20224 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
20231 AddToWorklist(Est.getNode());
20238 for (
int i = 0; i < Iterations; ++i) {
20240 AddToWorklist(NewEst.
getNode());
20243 AddToWorklist(NewEst.
getNode());
20246 AddToWorklist(NewEst.
getNode());
20249 AddToWorklist(Est.getNode());
20265 unsigned Iterations,
20267 EVT VT =
Arg.getValueType();
20274 AddToWorklist(HalfArg.
getNode());
20277 AddToWorklist(HalfArg.
getNode());
20280 for (
unsigned i = 0; i < Iterations; ++i) {
20282 AddToWorklist(NewEst.
getNode());
20285 AddToWorklist(NewEst.
getNode());
20288 AddToWorklist(NewEst.
getNode());
20291 AddToWorklist(Est.
getNode());
20297 AddToWorklist(Est.
getNode());
20309 unsigned Iterations,
20311 EVT VT =
Arg.getValueType();
20322 for (
unsigned i = 0; i < Iterations; ++i) {
20327 AddToWorklist(AEE.
getNode());
20330 AddToWorklist(RHS.
getNode());
20336 if (Reciprocal || (i + 1) < Iterations) {
20343 AddToWorklist(LHS.
getNode());
20346 AddToWorklist(Est.
getNode());
20361 EVT VT =
Op.getValueType();
20368 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
20375 bool UseOneConstNR =
false;
20379 AddToWorklist(Est.
getNode());
20382 Est = UseOneConstNR
20383 ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal)
20384 : buildSqrtNRTwoConst(
Op, Est, Iterations, Flags, Reciprocal);
20393 Attribute Denorms =
F.getFnAttribute(
"denormal-fp-math");
20402 Est = DAG.
getNode(SelOpcode, DL, VT, IsDenorm, FPZero, Est);
20403 AddToWorklist(Fabs.
getNode());
20404 AddToWorklist(IsDenorm.
getNode());
20405 AddToWorklist(Est.
getNode());
20410 Est = DAG.
getNode(SelOpcode, DL, VT, IsZero, FPZero, Est);
20411 AddToWorklist(IsZero.
getNode());
20412 AddToWorklist(Est.
getNode());
20423 return buildSqrtEstimateImpl(Op, Flags,
true);
20427 return buildSqrtEstimateImpl(Op, Flags,
false);
20431bool DAGCombiner::isAlias(
SDNode *Op0,
SDNode *Op1)
const {
20433 struct MemUseCharacteristics {
20441 auto getCharacteristics = [](
SDNode *
N) -> MemUseCharacteristics {
20442 if (
const auto *LSN = dyn_cast<LSBaseSDNode>(
N)) {
20444 if (
auto *
C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
20446 ?
C->getSExtValue()
20448 ? -1 *
C->getSExtValue()
20450 return {LSN->isVolatile(), LSN->getBasePtr(),
Offset ,
20452 LSN->getMemOperand()};
20454 if (
const auto *LN = cast<LifetimeSDNode>(
N))
20455 return {
false , LN->getOperand(1),
20456 (LN->hasOffset()) ? LN->getOffset() : 0,
20461 return {
false ,
SDValue(), (int64_t)0 ,
20465 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
20466 MUC1 = getCharacteristics(Op1);
20469 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
20470 MUC0.Offset == MUC1.Offset)
20474 if (MUC0.IsVolatile && MUC1.IsVolatile)
20477 if (MUC0.MMO && MUC1.MMO) {
20478 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
20479 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
20492 if (!MUC0.MMO || !MUC1.MMO)
20498 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
20499 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
20506 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
20507 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
20508 unsigned OrigAlignment0 = MUC0.MMO->getBaseAlignment();
20509 unsigned OrigAlignment1 = MUC1.MMO->getBaseAlignment();
20510 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
20511 MUC0.NumBytes.hasValue() && MUC1.NumBytes.hasValue() &&
20512 *MUC0.NumBytes == *MUC1.NumBytes && OrigAlignment0 > *MUC0.NumBytes) {
20513 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0;
20514 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1;
20518 if ((OffAlign0 + *MUC0.NumBytes) <= OffAlign1 ||
20519 (OffAlign1 + *MUC1.NumBytes) <= OffAlign0)
20532 if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue()) {
20534 int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
20535 int64_t Overlap0 = *MUC0.NumBytes + SrcValOffset0 - MinOffset;
20536 int64_t Overlap1 = *MUC1.NumBytes + SrcValOffset1 - MinOffset;
20552void DAGCombiner::GatherAllAliases(
SDNode *
N,
SDValue OriginalChain,
20558 const bool IsLoad = isa<LoadSDNode>(
N) && !cast<LoadSDNode>(
N)->isVolatile();
20562 unsigned Depth = 0;
20566 switch (
C.getOpcode()) {
20574 bool IsOpLoad = isa<LoadSDNode>(
C.getNode()) &&
20575 !cast<LSBaseSDNode>(
C.getNode())->isVolatile();
20576 if ((IsLoad && IsOpLoad) || !isAlias(
N,
C.getNode())) {
20578 C =
C.getOperand(0);
20587 C =
C.getOperand(0);
20594 if (!isAlias(
N,
C.getNode())) {
20596 C =
C.getOperand(0);
20609 while (!Chains.
empty()) {
20643 if (ImproveChain(Chain)) {
20665 GatherAllAliases(
N, OldChain, Aliases);
20668 if (Aliases.
size() == 0)
20672 if (Aliases.
size() == 1)
20681struct UnitT { } Unit;
20682bool operator==(
const UnitT &,
const UnitT &) {
return true; }
20683bool operator!=(
const UnitT &,
const UnitT &) {
return false; }
20699bool DAGCombiner::parallelizeChainedStores(
StoreSDNode *St) {
20716 if (!
BasePtr.getBase().getNode())
20720 if (
BasePtr.getBase().isUndef())
20730 if (Chain->isVolatile() || Chain->isIndexed())
20737 if (!
BasePtr.equalBaseIndex(Ptr, DAG, Offset))
20739 int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
20742 auto I = Intervals.find(Offset);
20744 if (
I != Intervals.end() &&
I.start() < (
Offset + Length))
20747 if (
I != Intervals.begin() && (--
I).stop() <=
Offset)
20749 Intervals.insert(Offset, Offset + Length, Unit);
20756 if (ChainedStores.
size() == 0)
20763 for (
unsigned I = ChainedStores.
size();
I;) {
20765 SDValue BetterChain = FindBetterChain(S, NewChain);
20769 ChainedStores[
I] = S;
20773 SDValue BetterChain = FindBetterChain(St, NewChain);
20788 auto hasImprovedChain = [&](
SDValue ST) ->
bool {
20789 return ST->getOperand(0) != NewChain;
20791 bool AddNewChain =
llvm::all_of(TFOps, hasImprovedChain);
20798 AddToWorklist(STChain);
20806bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
20813 if (!
BasePtr.getBase().getNode())
20817 if (
BasePtr.getBase().isUndef())
20821 if (parallelizeChainedStores(St))
20826 if (St->
getChain() != BetterChain) {
20827 replaceStoreChain(St, BetterChain);
20837 DAGCombiner(*
this, AA, OptLevel).Run(Level);
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static uint64_t * getMemory(unsigned numWords)
A utility function for allocating memory and checking for allocation failure.
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isLoad(int Opcode)
This file contains the simple types necessary to represent the attributes associated with functions a...
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques=false)
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x,...
static cl::opt< unsigned > TokenFactorInlineLimit("combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors"))
static bool isConstantOrConstantVector(SDValue N, bool NoOpaques=false)
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, SDValue ExtractFrom, SDValue &Mask, const SDLoc &DL)
Helper function for visitOR to extract the needed side of a rotate idiom from a shl/srl/mul/udiv.
static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG)
If a vector binop is performed on splat values, it may be profitable to extract, scalarize,...
static int numVectorEltsOrZero(EVT T)
static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, unsigned Depth=0)
If isNegatibleForFree returns true, return the newly negated expression.
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG)
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, SelectionDAG &DAG, const SDLoc &DL)
Given the operands of an add/sub operation, see if the 2nd operand is a masked 0/1 whose source opera...
static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf)
If we have a unary shuffle of a shuffle, see if it can be folded away completely.
static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N, APFloat(*Op)(const APFloat &, const APFloat &))
static cl::opt< bool > StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, cl::desc("Bypass the profitability model of load slicing"), cl::init(false))
Hidden option to stress test load slicing, i.e., when this option is enabled, load slicing bypasses m...
static cl::opt< bool > UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true), cl::desc("Enable DAG combiner's use of TBAA"))
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices.
static const Optional< ByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, bool Root=false)
Recursively traverses the expression calculating the origin of the requested byte of the given value.
static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector produced by a wide binary operator try to use a narrow binary operat...
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0....
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector from a wide vector load, convert to a narrow load to eliminate the e...
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, SelectionDAG &DAG)
static char isNegatibleForFree(SDValue Op, bool LegalOperations, const TargetLowering &TLI, const TargetOptions *Options, bool ForCodeSize, unsigned Depth=0)
Return 1 if we can compute the negated form of the specified expression for the same cost as the expr...
static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, SDValue X, SDValue Carry0, SDValue Carry1, SDNode *N)
If we are facing some sort of diamond carry propapagtion pattern try to break it up to generate somet...
static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If a shuffle inserts exactly one element from a source vector operand into another vector operand and...
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType)
static cl::opt< bool > CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, cl::desc("Enable DAG combiner's use of IR alias analysis"))
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, const TargetLowering &TLI)
Return true if 'Use' is a load or a store that uses N as its base pointer and that N may be folded in...
static unsigned BigEndianByteAt(unsigned BW, unsigned i)
static SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N)
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations)
Transform a vector binary operation into a scalar binary operation by moving the math/logic after an ...
static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static std::pair< SDValue, SDValue > SplitVSETCC(const SDNode *N, SelectionDAG &DAG)
static SDValue stripTruncAndExt(SDValue Value)
static bool isContractable(SDNode *N)
static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles followed by concatenatio...
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, SelectionDAG &DAG)
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out.
static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG)
Try to fold a 'not' shifted sign-bit with add/sub with constant operand into a shift and add with a d...
static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef< int > Mask)
If the shuffle mask is taking exactly one element from the first vector operand and passing through a...
static cl::opt< bool > MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads"))
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG)
static SDNode * ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo.
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS, const TargetLowering &TLI)
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, const TargetLowering &TLI, bool Force)
Flips a boolean if it is cheaper to compute.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, KnownBits &Known)
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V)
static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static Optional< bool > isBigEndian(const SmallVector< int64_t, 4 > &ByteOffsets, int64_t FirstOffset)
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
Generate Min/Max node.
static unsigned LittleEndianByteAt(unsigned BW, unsigned i)
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N)
OR combines for which the commuted variant will be tried as well.
static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG, bool LegalOperations)
static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask)
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue flipBoolean(SDValue V, const SDLoc &DL, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG)
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations)
static cl::opt< std::string > CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden, cl::desc("Only use DAG-combiner alias analysis in this" " function"))
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap.
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static ManagedStatic< DebugCounter > DC
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
static Value * simplifyDivRem(Value *Op0, Value *Op1, bool IsDiv)
Check for common or similar folds of integer division or integer remainder.
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
static bool hasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB)
The main low level interface to the alias analysis implementation.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
const fltSemantics & getSemantics() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set a given bit to 1.
APInt abs() const
Get the absolute value;.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getNullValue(unsigned numBits)
Get the '0' value.
unsigned getBitWidth() const
Return the number of bits in the APInt.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
bool isOneValue() const
Determine if this is a value of 1.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
unsigned logBase2() const
bool isAllOnesValue() const
Determine if all bits are set.
bool isNullValue() const
Determine if all bits are clear.
bool getBoolValue() const
Convert APInt to a boolean value.
bool isMask(unsigned numBits) const
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
APInt zextOrSelf(unsigned width) const
Zero extend or truncate to width.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
unsigned countTrailingOnes() const
Count the number of trailing one bits.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
StringRef getValueAsString() const
Return the attribute's value as a string.
Helper struct to parse and store a memory address as base + index + offset.
bool contains(const SelectionDAG &DAG, int64_t BitSize, const BaseIndexOffset &Other, int64_t OtherBitSize, int64_t &BitOffset) const
static bool computeAliasing(const SDNode *Op0, const Optional< int64_t > NumBytes0, const SDNode *Op1, const Optional< int64_t > NumBytes1, const SelectionDAG &DAG, bool &IsAlias)
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG, int64_t &Off) const
A "pseudo-class" with methods for operating on BUILD_VECTORs.
ISD::CondCode get() const
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
bool isZero() const
Return true if the value is positive or negative zero.
ConstantFP - Floating Point Values [float, double].
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX)
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
bool isAllOnesValue() const
This is an important base class in LLVM.
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
A parsed version of the target data layout string in and methods for querying it.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
bool isLittleEndian() const
Layout endianness...
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This class is used to form a handle around another node that is persistent and is updated across invo...
NodeT & get() const
get - Dereference as a NodeT reference.
This is an important class for using LLVM in a threaded context.
Base class for LoadSDNode and StoreSDNode.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getSizeInBits() const
static MVT getIntegerVT(unsigned BitWidth)
static mvt_range all_valuetypes()
SimpleValueType Iteration.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
const Function & getFunction() const
Return the LLVM function that this machine code represents.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
const SDValue & getIndex() const
const SDValue & getScale() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
This class is used to represent an MLOAD node.
const SDValue & getBasePtr() const
bool isExpandingLoad() const
const SDValue & getMask() const
const SDValue & getPassThru() const
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
bool isDereferenceable() const
unsigned getAlignment() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< value_op_iterator > op_values() const
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
const SDNodeFlags getFlags() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
unsigned getScalarValueSizeInBits() const
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
bool isKnownNeverZero(SDValue Op) const
Test whether the given SDValue is known to contain non-zero value(s).
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N)
Test whether the given value is a constant int or similar node.
unsigned InferPtrAlignment(SDValue Ptr) const
Infer alignment of a load / store address.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 node can overflow.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts)
Test whether V has a splatted value for all the demanded elements.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, const SDLoc &DL)
Returns sum of the base pointer and offset.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits)
See if the specified operand can be simplified with the knowledge that only the bits specified by Dem...
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags=SDNodeFlags())
Get the specified node if it's already available, or else return NULL.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, SDNode *N1, SDNode *N2)
bool isKnownToBeAPowerOfTwo(SDValue Val) const
Test if the given value is known to have exactly one bit set.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, const SDNodeFlags Flags=SDNodeFlags())
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
void Combine(CombineLevel Level, AliasAnalysis *AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue is known to never be NaN.
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
SDValue makeEquivalentMemoryOrdering(LoadSDNode *Old, SDValue New)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
A vector that has set insertion semantics.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
LLVM_NODISCARD T pop_back_val()
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
LLVM_NODISCARD bool empty() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
LLVM_NODISCARD bool empty() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
LLVM_NODISCARD T pop_back_val()
reference emplace_back(ArgTypes &&... Args)
iterator erase(const_iterator CI)
typename SuperClass::const_iterator const_iterator
iterator insert(iterator I, T &&Elt)
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
void reassociateOps(MachineInstr &Root, MachineInstr &Prev, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const
Attempt to reassociate \P Root and \P Prev according to \P Pattern to reduce critical path length.
bool has(LibFunc F) const
Tests whether a library function is available.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool hasPairedLoad(EVT, unsigned &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always beneficiates from combining into FMA for a given value type.
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
virtual bool decomposeMulByConstant(EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool hasBitPreservingFPLogic(EVT VT) const
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool reduceSelectOfFPConstantLoads(bool IsFPSetCC) const
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
unsigned getGatherAllAliasesMaxDepth() const
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isNarrowingProfitable(EVT, EVT) const
Return true if it's profitable to narrow operations of type VT1 to VT2.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual bool isFMAFasterThanFMulAndFAdd(EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const SelectionDAG &DAG) const
Returns if it's reasonable to merge stores to MemVT size.
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, unsigned Alignment=1, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
bool isConstFalseVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored.
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isConstTrueVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount though its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned NoInfsFPMath
NoInfsFPMath - This flag is enabled when the -enable-no-infs-fp-math flag is specified on the command...
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine,...
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
User * getUser() const LLVM_READONLY
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one user of this value.
iterator_range< use_iterator > uses()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
@ BSWAP
Byte Swap and Counting operators.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions are non-strict, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ ADDCARRY
Carry-using nodes for multiple precision addition and subtraction.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ FP_ROUND_INREG
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ CARRY_FALSE
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, bool isInteger)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, bool isInteger)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
This class represents lattice values for constants.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
bool operator<(int64_t V1, const APSInt &V2)
const Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool operator==(uint64_t V1, const APInt &V2)
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
bool operator!=(uint64_t V1, const APInt &V2)
bool operator>=(int64_t V1, const APSInt &V2)
std::string & operator+=(std::string &buffer, StringRef string)
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 maximum semantics.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool operator>(int64_t V1, const APSInt &V2)
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
void sort(IteratorTy Start, IteratorTy End)
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
detail::ValueMatchesPoly< M > HasValue(M Matcher)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
std::enable_if<!is_simple_type< Y >::value, typenamecast_retty< X, constY >::ret_type >::type cast(const Y &Val)
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
bool isNullOrNullSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
@ Z
zlib style complession
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
auto count_if(R &&Range, UnaryPredicate P) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
APFloat neg(APFloat X)
Returns the negated value of the argument.
bool isAllOnesOrAllOnesSplat(SDValue V)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
AliasResult
The possible results of an alias query.
@ NoAlias
The two locations do not alias at all.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 minimum semantics.
bool isOneOrOneSplat(SDValue V)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
bool operator<=(int64_t V1, const APSInt &V2)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static unsigned int semanticsPrecision(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
unsigned getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
unsigned getScalarSizeInBits() const
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
unsigned getBitWidth() const
Get the bit width of this value.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
RetVal visit(const SCEV *S)
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
bool hasApproximateFuncs() const
bool hasAllowReciprocal() const
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
bool hasVectorReduction() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
void AddToWorklist(SDNode *N)
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)
hash_code combine(size_t length, char *buffer_ptr, char *buffer_end, const T &arg, const Ts &...args)
Recursive, variadic combining method.